HfstTransducer.h

Go to the documentation of this file.

// Copyright (c) 2016 University of Helsinki
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 3 of the License, or (at your option) any later version.
// See the file COPYING included with this distribution for more
// information.

#ifndef _HFST_TRANSDUCER_H_
#define _HFST_TRANSDUCER_H_

#if HAVE_CONFIG_H
#  include <config.h>
#endif

#include "FormatSpecifiers.h"

#include "HfstDataTypes.h"
#include "HfstSymbolDefs.h"
#include "implementations/HfstTransitionGraph.h"
#include "parsers/LexcCompiler.h"

#if HAVE_SFST
#include "implementations/SfstTransducer.h"
#endif

#if HAVE_OPENFST
#include "implementations/TropicalWeightTransducer.h"
#include "implementations/LogWeightTransducer.h"
#endif

#if HAVE_FOMA
#include "implementations/FomaTransducer.h"
#endif

#if HAVE_XFSM
#include "implementations/XfsmTransducer.h"
#endif

/* Include here the header file of the files that take care 
   of the interaction between HFST and your transducer library. */
//#if HAVE_MY_TRANSDUCER_LIBRARY
//#include "implementations/MyTransducerLibraryTransducer.h"
//#endif

#include "implementations/HfstOlTransducer.h"
#include "HfstTokenizer.h"
#include "implementations/ConvertTransducerFormat.h"
#include "HfstExceptionDefs.h"
#include "HfstInputStream.h"
#include "HfstOutputStream.h"

#include <string>
#include <cassert>
#include <iostream>
#include <vector>
#include <map>
#include <set>

#include "hfstdll.h"

namespace hfst
{
  namespace implementations {    
    template <class T> class HfstTransitionGraph;
    class HfstTropicalTransducerTransitionData;
    typedef HfstTransitionGraph<HfstTropicalTransducerTransitionData> 
      HfstBasicTransducer; 
  }
  class HfstCompiler;
  class HfstTransducer;
  namespace xfst {
    class XfstCompiler;
  }
  namespace xeroxRules
  {
      class Rule;
      HfstTransducer bracketedReplace( const Rule &rule, bool optional);
  }

  using hfst::implementations::HfstOlTransducer;

#if HAVE_SFST
  using hfst::implementations::SfstTransducer;
#endif // #if HAVE_SFST

#if HAVE_OPENFST
  using hfst::implementations::TropicalWeightTransducer;
  using hfst::implementations::TropicalWeightState;
  using hfst::implementations::TropicalWeightStateIterator;
#if HAVE_OPENFST_LOG
  using hfst::implementations::LogWeightTransducer;
#endif // #if HAVE_OPENFST_LOG
#endif // #if HAVE_OPENFST

#if HAVE_FOMA
  using hfst::implementations::FomaTransducer;
#endif // #if HAVE_FOMA

#if HAVE_XFSM
  using hfst::implementations::XfsmTransducer;
#endif // #if HAVE_XFSM

  /* Add here the transducer class of your transducer library. */
  //#if HAVE_MY_TRANSDUCER_LIBRARY
  //  using hfst::implementations::MyTransducerLibraryTransducer;
  //#endif // #if HAVE_MY_TRANSDUCER_LIBRARY

  // *** TESTING AND OPTIMIZATION...

#if HAVE_XFSM
  HFSTDLL void initialize_xfsm();

  class InitializeXfsm
  {
  public:
    HFSTDLL InitializeXfsm();
  };
#endif

  enum MinimizationAlgorithm { HOPCROFT, BRZOZOWSKI };
  /* Which minimization algorithm is used. 
     In foma, Hopcroft is always used. 
     In OpenFst and SFST, the default algorithm is Hopcroft. */
  HFSTDLL void set_minimization_algorithm(MinimizationAlgorithm);
  HFSTDLL MinimizationAlgorithm get_minimization_algorithm(); 

  HFSTDLL void set_encode_weights(bool);
  HFSTDLL bool get_encode_weights();

  HFSTDLL void set_minimize_even_if_already_minimal(bool);
  HFSTDLL bool get_minimize_even_if_already_minimal();

  HFSTDLL void set_xerox_composition(bool);
  HFSTDLL bool get_xerox_composition();

  HFSTDLL void set_flag_is_epsilon_in_composition(bool);
  HFSTDLL bool get_flag_is_epsilon_in_composition();

  /* Whether in harmonization the smaller transducer is always harmonized
     according to the bigger transducer. */
  HFSTDLL void set_harmonize_smaller(bool);
  HFSTDLL bool get_harmonize_smaller();

  /* Whether unknown and identity symbols are used. By default, they are used.
     These symbols are always reserved for use and included in alphabets 
     of transducers, but code optimization is possible if it is known 
     that they do not appear in transducer transitions. */
  HFSTDLL void set_unknown_symbols_in_use(bool);
  HFSTDLL bool get_unknown_symbols_in_use();

  void set_warning_stream(std::ostream * os);
  std::ostream * get_warning_stream();

  // *** ...TESTING AND OPTIMIZATION


  class HfstTransducer
  {

    // ***** PROTECTED VARIABLES AND INTERFACE *****

  protected:

    /* The backend implementation type of the transducer */
    ImplementationType type; 

    bool anonymous;    // currently not used
    bool is_trie;      // currently not used
    std::string name;  /* The name of the transducer */
    std::map<std::string,std::string> props;    // rest of fst metadata
    /* The union of possible backend implementations. */
    union TransducerImplementation
    {
#if HAVE_SFST
      hfst::implementations::Transducer * sfst;
#endif
#if HAVE_OPENFST
      hfst::implementations::StdVectorFst * tropical_ofst;
#if HAVE_OPENFST_LOG
      hfst::implementations::LogFst * log_ofst;
#endif
#endif
#if HAVE_FOMA
      fsm * foma;
#endif
#if HAVE_XFSM
      NETptr xfsm;
#endif

      /* Add here your own transducer backend implementation. */
      //#if HAVE_MY_TRANSDUCER_LIBRARY
      //      hfst::implementations::MyFst * my_transducer_library;
      //#endif

      hfst_ol::Transducer * hfst_ol;

#if HAVE_OPENFST // is this needed?
      hfst::implementations::StdVectorFst * internal;
#endif 
    };

    /* The backend implementation */
    TransducerImplementation implementation;

    /* Interfaces through which the backend implementations can be accessed */
#if HAVE_SFST
    static hfst::implementations::SfstTransducer sfst_interface;
#endif
#if HAVE_OPENFST
    static hfst::implementations::TropicalWeightTransducer 
      tropical_ofst_interface;
#if HAVE_OPENFST_LOG
    static hfst::implementations::LogWeightTransducer log_ofst_interface;
#endif
#endif
#if HAVE_FOMA
    static hfst::implementations::FomaTransducer foma_interface;
#endif
    static hfst::implementations::HfstOlTransducer hfst_ol_interface;
#if HAVE_XFSM
    static hfst::implementations::XfsmTransducer xfsm_interface;
#endif

    /* Add here the class that takes care of the interaction between
       HFST and your transducer library. */
    //#if HAVE_MY_TRANSDUCER_LIBRARY
    //static hfst::implementations::MyTransducerLibraryTransducer
    //  my_transducer_library_interface;
    //#endif

    /* The same as harmonize but does not change \a another but
       returns a harmonized version of that. If this and \a another 
       have type FOMA_TYPE, NULL is returned. */
    HfstTransducer * harmonize_(const HfstTransducer &another);

    HfstTransducer * harmonize_symbol_encodings(const HfstTransducer &another);

    /* Check if transducer \a another has in its alphabet flag diacritics
       that are not found in the alphabet of this transducer and insert
       all missing flag diacritics to \a missing_flags. 
       \a return_on_first_miss defines whether the function returns as soon as
       a missing flag is found so that only that flag is inserted to
       \a missing flags. */
    bool check_for_missing_flags_in(const HfstTransducer &another,
                                    StringSet &missing_flags,
                                    bool return_on_first_miss) const;

    /* Disjunct trie transducers efficiently so that the result is also
       a trie.
       Currently not implemented, TODO */
    HfstTransducer &disjunct_as_tries(HfstTransducer &another,
                                      ImplementationType type);  


    /* Remove paths where @..._2@ transitions immediately preceede 
       @..._1@ transitions, i.e. transitions resulting from incorrect
       ordering of flag diacritics. */
    HfstTransducer &remove_illegal_flag_paths(void);


    /* For internal use */
    static HfstTransducer &read_in_att_format
      (FILE *ifile, ImplementationType type, const std::string &epsilon_symbol);

  public:
    /* Whether the conversion requested can be done without losing 
       any information */
    static bool is_safe_conversion(ImplementationType original,
                                   ImplementationType conversion);

    /* whether HFST is linked to the transducer library 
       needed by implementation type \a type. */
    HFSTDLL static bool is_implementation_type_available(ImplementationType type);

    HFSTDLL unsigned int number_of_states() const;
    HFSTDLL unsigned int number_of_arcs() const;

    HFSTDLL void twosided_flag_diacritics();

  protected:
    /* For internal use: Create a tokenizer that recognizes all symbols 
       that occur in the transducer. */
    HfstTokenizer create_tokenizer();

    /* For internal use. Implemented only for SFST_TYPE.
       Get all symbol pairs that occur in the transitions of the transducer. */
    StringPairSet get_symbol_pairs();

  protected:

    /* Get the number used to represent the symbol \a symbol. */
    unsigned int get_symbol_number(const std::string &symbol);
    
    /* For internal use, implemented only for SFST_TYPE. */          
    std::vector<HfstTransducer*> extract_path_transducers();

    /* For internal use:
       Create a new transducer equivalent to \a t in format \a type. */
    static HfstTransducer &convert
      (const HfstTransducer &t, ImplementationType type);

    /* For internal use:
       Create an HfstBasicTransducer equivalent to \a t end delete
       the backend implementation of \a t. */
    implementations::HfstBasicTransducer * convert_to_basic_transducer();

    /* For internal use:
       Create an HfstBasicTransducer equivalent to \a t. */
    implementations::HfstBasicTransducer * get_basic_transducer() const;

    /* For internal use:
       Create a backend implementation of the same type that this transducer
       has and that is equivalent to \a t and delete \a t. Assign the 
       backend implementation as the value of the implementation of this
       transducer. */
    HfstTransducer &convert_to_hfst_transducer
      (implementations::HfstBasicTransducer *t);

    /* \brief For internal use: Create a transducer of type \a type as 
       defined in AT&T format in file named \a filename.
       \a epsilon_symbol defines how epsilons are represented.

        NOTE: If the transition symbols contain space characters
        they must be represented as "@_SPACE_@" because
        whitespace characters are used as field separators 
        in AT&T format.

       @pre The file exists, otherwise an exception is thrown.
       @see HfstTransducer(FILE, ImplementationType, const std::string&)
       @throws StreamNotReadableException 
       @throws NotValidAttFormatException 
    */
    static HfstTransducer &read_in_att_format
      (const std::string &filename, ImplementationType type, 
       const std::string &epsilon_symbol);

  public:
    HFSTDLL static HfstTransducer * prolog_file_to_xfsm_transducer(const char * filename);


    /* For debugging */
  public:
    HFSTDLL void print_alphabet();
  protected:
    static float get_profile_seconds(ImplementationType type);

#include "hfst_apply_schemas.h"



    // ***** THE PUBLIC INTERFACE *****

  public:


    // ------------------------------------------------
    // ----- Constructors, destructor, assignment -----
    // ------------------------------------------------

    HFSTDLL HfstTransducer();

    HFSTDLL HfstTransducer(ImplementationType type);

    HFSTDLL HfstTransducer(const std::string& utf8_str, 
           const HfstTokenizer &multichar_symbol_tokenizer,
                   ImplementationType type);

    HFSTDLL HfstTransducer(const std::string& input_utf8_str,
                   const std::string& output_utf8_str,
                   const HfstTokenizer &multichar_symbol_tokenizer,
                   ImplementationType type);

    /* @brief Create a transducer that recognizes the union of string pairs in 
       \a sps. The type of the transducer is defined by \a type. \a cyclic
       defines whether the transducer recognizes any number (from zero to
       infinity, inclusive) of consecutive string pairs in \s sps. */
    HFSTDLL HfstTransducer(const StringPairSet & sps, ImplementationType type, 
                   bool cyclic=false);

    /* \brief Create a transducer that recognizes the concatenation of
       string pairs in \a spv. The type of the transducer is defined
       by \a type. */
    HFSTDLL HfstTransducer(const StringPairVector & spv, ImplementationType type);

    /* \brief Create a transducer that recognizes the concatenation of the 
       unions of string pairs in string pair sets in \a spsv. The type of
       the transducer is defined by \a type. */
    HFSTDLL HfstTransducer(const std::vector<StringPairSet> & spsv, 
                   ImplementationType type);

    HFSTDLL HfstTransducer(HfstInputStream &in);

    HFSTDLL HfstTransducer(const HfstTransducer &another);

    HFSTDLL HfstTransducer(const hfst::implementations::HfstBasicTransducer &t, 
                   ImplementationType type);

    HFSTDLL HfstTransducer(const std::string &symbol, ImplementationType type);

    HFSTDLL HfstTransducer(const std::string &isymbol, const std::string &osymbol, 
                           ImplementationType type);

    HFSTDLL HfstTransducer(FILE * ifile, ImplementationType type, 
                   const std::string &epsilon_symbol, unsigned int & linecount);

    HFSTDLL HfstTransducer(FILE * ifile, ImplementationType type, 
                   const std::string &epsilon_symbol);


    HFSTDLL virtual ~HfstTransducer(void);

    HFSTDLL HfstTransducer &operator=(const HfstTransducer &another);

    HFSTDLL HfstTransducer &assign(const HfstTransducer &another);

    // ------------------------------------------------------------
    // ----------- Properties, comparison, conversion -------------
    // ------------------------------------------------------------

    HFSTDLL void set_name(const std::string &name);

    HFSTDLL std::string get_name() const;

    HFSTDLL void set_property(const std::string& property, const std::string& value);
    
    HFSTDLL std::string get_property(const std::string& property) const;
    HFSTDLL const std::map<std::string,std::string>& get_properties() const;
    HFSTDLL StringSet get_alphabet() const;

    HFSTDLL StringSet get_first_input_symbols() const;

    HFSTDLL void harmonize(HfstTransducer &another);

    HFSTDLL void insert_to_alphabet(const std::string &symbol); 
    HFSTDLL void insert_to_alphabet(const std::set<std::string> &symbols); 

    HFSTDLL void remove_from_alphabet(const std::string &symbol);
    HFSTDLL void remove_from_alphabet(const std::set<std::string> &symbols);

    // For XFSM format
    HFSTDLL void remove_symbols_from_alphabet(const StringSet & symbols);


    HFSTDLL HfstTransducer &prune_alphabet(bool force=true);

    HFSTDLL bool is_cyclic(void) const;

    HFSTDLL bool is_automaton(void) const;

    HFSTDLL ImplementationType get_type(void) const;

    HFSTDLL bool compare(const HfstTransducer &another, bool harmonize=true) const;

    HFSTDLL HfstTransducer &convert(ImplementationType type, std::string options="");


    // --------------------------------------------------------
    // --- String lookup and conversion to/from AT&T format ---
    // --------------------------------------------------------

    HFSTDLL void write_in_att_format(FILE * ofile, bool write_weights=true) const;

    HFSTDLL void write_in_att_format(char * buffer, bool write_weights=true) const;

    /* For XFSM_TYPE. */
    HFSTDLL void write_xfsm_transducer_in_att_format(const char * filename) const;
    HFSTDLL void write_xfsm_transducer_in_prolog_format(const char * filename) const;

    /* For other types. */
    HFSTDLL void write_in_prolog_format(FILE * file, const std::string & name,
                                bool write_weights=true);

    HFSTDLL void write_in_att_format_number
      (FILE * ofile, bool write_weights=true) const;


    HFSTDLL void write_in_att_format(const std::string &filename, 
                                     bool write_weights=true) const;


  public:
    /* \brief Call \a callback with some or all string pairs recognized 
       by the transducer?

        If the callback returns false the search will be terminated. 
        The \a cycles parameter
        indicates how many times a cycle will be followed, with negative numbers
        indicating unlimited. Note that if the transducer is cyclic and 
        cycles aren't capped,
        the search will not end until the callback returns false. */
    HFSTDLL void extract_paths(ExtractStringsCb& callback, int cycles=-1) const;

    HFSTDLL void extract_paths
      (HfstTwoLevelPaths &results, int max_num=-1, int cycles=-1) const;

    HFSTDLL void extract_random_paths
      (HfstTwoLevelPaths &results, int max_num) const;

    HFSTDLL void extract_random_paths_fd
      (HfstTwoLevelPaths &results, int max_num, bool filter_fd) const;

    /* \brief Call \a callback with extracted strings that are not 
       invalidated by flag diacritic rules.

       @see extract_paths(HfstTwoLevelPaths&, int, int) */
    HFSTDLL void extract_paths_fd
      (ExtractStringsCb& callback, int cycles=-1, bool filter_fd=true) const;

    // todo: handle flag diacritics
    // todo: throw TransducerIsCyclicException, if cyclic
    HFSTDLL void extract_shortest_paths
      (HfstTwoLevelPaths &results) const;
    
    HFSTDLL bool extract_longest_paths
      (HfstTwoLevelPaths &results, bool obey_flags=true /*,bool show_flags=false*/) const;

    HFSTDLL int longest_path_size(bool obey_flags=true) const;

  public:
    HFSTDLL void extract_paths_fd
      (HfstTwoLevelPaths &results, int max_num=-1, int cycles=-1, 
       bool filter_fd=true) const;

    HFSTDLL HfstOneLevelPaths * lookup(const StringVector& s,
                                       ssize_t limit = -1,
                                       double time_cutoff = 0.0) const;

    HFSTDLL HfstOneLevelPaths * lookup(const std::string & s,
                                       ssize_t limit = -1,
                                       double time_cutoff = 0.0) const;

    HFSTDLL HfstOneLevelPaths * lookup_fd(const StringVector& s,
                                          ssize_t limit = -1,
                                          double time_cutoff = 0.0) const;

    HFSTDLL HfstOneLevelPaths * lookup_fd(const std::string& s,
                                          ssize_t limit = -1,
                                          double time_cutoff = 0.0) const;

      HFSTDLL HfstOneLevelPaths * lookup(const HfstTokenizer& tok,
                                         const std::string &s,
                                         ssize_t limit = -1,
                                         double time_cutoff = 0.0) const;

    HFSTDLL HfstOneLevelPaths * lookup_fd(
        const HfstTokenizer& tok,
        const std::string &s, ssize_t limit = -1,
        double time_cutoff = 0.0) const;

    HFSTDLL HfstOneLevelPaths * lookdown(const StringVector& s,
                 ssize_t limit = -1) const;

    HFSTDLL HfstOneLevelPaths * lookdown(const std::string& s,
                 ssize_t limit = -1) const;

    HFSTDLL HfstOneLevelPaths * lookdown_fd(StringVector& s,
                    ssize_t limit = -1) const;

    HFSTDLL HfstOneLevelPaths * lookdown_fd(const std::string& s,
                    ssize_t limit = -1) const;

    HFSTDLL bool is_lookup_infinitely_ambiguous(const StringVector & s) const;
    HFSTDLL bool is_lookup_infinitely_ambiguous(const std::string & s) const;

    HFSTDLL bool is_lookdown_infinitely_ambiguous(const StringVector& s) const;

    HFSTDLL bool is_infinitely_ambiguous() const ;


    // -------------------------------------------
    // --------- Optimization operations ---------
    // -------------------------------------------

    HFSTDLL HfstTransducer &eliminate_flags();
    HFSTDLL HfstTransducer &eliminate_flag(const std::string &flag);

    HFSTDLL HfstTransducer &remove_epsilons();

    HFSTDLL HfstTransducer &prune();

    HFSTDLL HfstTransducer &determinize();

    HFSTDLL HfstTransducer &minimize();

    HFSTDLL HfstTransducer &n_best(unsigned int n);


    // ------------------------------------------------
    // ------------- Algebraic operations -------------
    // ------------------------------------------------

    HFSTDLL HfstTransducer &repeat_star();

    HFSTDLL HfstTransducer &repeat_plus();

    HFSTDLL HfstTransducer &repeat_n(unsigned int n);

    HFSTDLL HfstTransducer &repeat_n_minus(unsigned int n);

    HFSTDLL HfstTransducer &repeat_n_plus(unsigned int n);

    HFSTDLL HfstTransducer& repeat_n_to_k(unsigned int n, unsigned int k);

    HFSTDLL HfstTransducer &optionalize();

    HFSTDLL HfstTransducer &invert();

    HFSTDLL HfstTransducer &reverse();

    HFSTDLL HfstTransducer &input_project();

    HFSTDLL HfstTransducer &output_project();

    HFSTDLL HfstTransducer &compose(const HfstTransducer &another, 
                            bool harmonize=true);

    HFSTDLL HfstTransducer &merge(const HfstTransducer &another, const std::map<std::string, std::set<std::string> > & list_symbols);

    HFSTDLL HfstTransducer &merge(const HfstTransducer &another, const struct hfst::xre::XreConstructorArguments & args);

    HFSTDLL HfstTransducer &compose_intersect(const HfstTransducerVector &v,
                                      bool invert=false, bool harmonize=true);

    HFSTDLL HfstTransducer &concatenate(const HfstTransducer &another, bool harmonize=true);

    HFSTDLL HfstTransducer &disjunct(const HfstTransducer &another, bool harmonize=true);

    HFSTDLL HfstTransducer &priority_union(const HfstTransducer &another);


    HFSTDLL HfstTransducer &lenient_composition(const HfstTransducer &another, bool harmonize=true);

    HFSTDLL HfstTransducer &cross_product(const HfstTransducer &another, bool harmonize=true);


    /*
     *  \brief Shuffle this transducer with transducer \@ another.
     *
     *  If transducer A accepts string "foo" and transducer B string "bar",
     *  the transducer that results from shuffling A and B accepts all strings
     *  [(f|b)(o|a)(o|r)].
     *  
     *  @pre Both transducers must be automata, i.e. map strings onto themselves.
     *
     */
    HFSTDLL HfstTransducer &shuffle(const HfstTransducer &another, bool harmonize=true);

    HFSTDLL static HfstTransducer universal_pair ( ImplementationType type );

    HFSTDLL static HfstTransducer identity_pair ( ImplementationType type );





    /* For HfstCompiler: Optimized disjunction function. */
    HFSTDLL HfstTransducer &disjunct(const StringPairVector &spv);

    HFSTDLL HfstTransducer &intersect(const HfstTransducer &another, bool harmonize=true);

    HFSTDLL HfstTransducer &subtract(const HfstTransducer &another, bool harmonize=true);


    // ------------------------------------------------
    // ---------- Insertion and substitution ----------
    // ------------------------------------------------

    HFSTDLL HfstTransducer &insert_freely(const StringPair &symbol_pair, bool harmonize=true);

    HFSTDLL HfstTransducer &insert_freely(const HfstTransducer &tr, bool harmonize=true);

  HFSTDLL HfstTransducer &substitute
  (bool (*func)(const StringPair &sp, StringPairSet &sps));

    HFSTDLL HfstTransducer &substitute(const std::string &old_symbol,
                               const std::string &new_symbol,
                               bool input_side=true,
                               bool output_side=true);
                                                      
    HFSTDLL HfstTransducer &substitute(const StringPair &old_symbol_pair,
                               const StringPair &new_symbol_pair);

    HFSTDLL HfstTransducer &substitute(const StringPair &old_symbol_pair,
                               const StringPairSet &new_symbol_pair_set);

    HFSTDLL HfstTransducer & substitute_symbol(const std::string &old_symbol, const std::string &new_symbol, bool input_side=true, bool output_side=true);
    HFSTDLL HfstTransducer & substitute_symbol_pair(const StringPair &old_symbol_pair, const StringPair &new_symbol_pair);
    HFSTDLL HfstTransducer & substitute_symbol_pair_with_set(const StringPair &old_symbol_pair, const hfst::StringPairSet &new_symbol_pair_set);
    HFSTDLL HfstTransducer & substitute_symbol_pair_with_transducer(const StringPair &symbol_pair, HfstTransducer &transducer, bool harmonize=true);


    HFSTDLL HfstTransducer &substitute(const HfstSymbolSubstitutions &substitutions);

    HFSTDLL HfstTransducer &substitute_symbols(const HfstSymbolSubstitutions &substitutions);

    HFSTDLL HfstTransducer &substitute(const HfstSymbolPairSubstitutions &substitutions);

    HFSTDLL HfstTransducer &substitute_symbol_pairs(const HfstSymbolPairSubstitutions &substitutions);

    HFSTDLL HfstTransducer &substitute(const StringPair &symbol_pair,
                               HfstTransducer &transducer, bool harmonize=true);

    // -----------------------------------------------
    // --------------- Weight handling --------------- 
    // -----------------------------------------------

    HFSTDLL HfstTransducer &set_final_weights(float weight, bool increment=false);

    HFSTDLL HfstTransducer &transform_weights(float (*func)(float));

    HFSTDLL HfstTransducer &push_weights(PushType type);


    HFSTDLL static HfstTransducer * read_lexc_ptr(const std::string &filename,
                                          ImplementationType type,
                                          bool verbose);

    HFSTDLL static HfstTransducer read_lexc(const std::string &filename,
                                    ImplementationType type,
                                    bool verbose);

    // *** For commandline programs. ***

    /* For each flag diacritic fd that is included in the alphabet of
       transducer \a another but not in the alphabet of this transducer,
       insert freely a transition fd:fd in this transducer. */
    HFSTDLL void insert_freely_missing_flags_from
      (const HfstTransducer &another);

    /*
      If both \a this and \a another contain flag diacritics, replace flag
      diacritic @X.Y.(.Z)@ by @X.Y_1(.Z)@ in \a this and replace it by
      @X.Y_2(.Z)@ in \a another. 
      
      If \a insert_renamed_flags is true, then the flags from \a this are 
      inserted freely in \a another and vice versa after replacing.
     */
    HFSTDLL void harmonize_flag_diacritics(HfstTransducer &another,
                                   bool insert_renamed_flags=true);
    
    HFSTDLL void insert_missing_symbols_to_alphabet_from(const HfstTransducer &another, bool only_special_symbols=false);

    HFSTDLL StringSet insert_missing_diacritics_to_alphabet_from(const HfstTransducer &another);

    HFSTDLL static bool is_special_symbol(const std::string & symbol);
    
    /* Whether the alphabet of transducer \a another includes flag diacritics
       that are not included in the alphabet of this transducer. */
    HFSTDLL bool check_for_missing_flags_in(const HfstTransducer &another) const;

    /* Return true if \a this has flag diacritics in the alphabet. */
    HFSTDLL bool has_flag_diacritics(void) const;

    

    // *** Friends **** //

    HFSTDLL friend std::ostream& operator<<(std::ostream &out, const HfstTransducer &t);
    HFSTDLL friend std::ostream& redirect(std::ostream &out, const HfstTransducer &t);
    friend class HfstInputStream;
    friend class HfstOutputStream;
    friend class hfst::implementations::HfstTransitionGraph<class C>;
    friend class HfstCompiler;
    friend class hfst::implementations::ConversionFunctions;
    friend class HfstGrammar;
    friend class xfst::XfstCompiler;
    //friend HfstTransducer bracketedReplace( const hfst::xeroxRules::Rule &rule, bool optional);
    friend hfst::HfstTransducer hfst::xeroxRules::bracketedReplace(const hfst::xeroxRules::Rule&, bool);
  };

  HFSTDLL std::ostream &operator<<(std::ostream &out,const HfstTransducer &t);

  HFSTDLL std::ostream &redirect(std::ostream &out,const HfstTransducer &t);

  namespace rules
  {
    enum ReplaceType {REPL_UP, REPL_DOWN, REPL_RIGHT, REPL_LEFT,
              REPL_DOWN_KARTTUNEN};
    enum TwolType {twol_right, twol_left, twol_both};

    /* helping methods */
    HFSTDLL HfstTransducer universal_fst
      (const StringPairSet &alphabet, ImplementationType type);
    HFSTDLL HfstTransducer negation_fst
      (const HfstTransducer &t, const StringPairSet &alphabet);

    HFSTDLL HfstTransducer replace
      (HfstTransducer &t, ReplaceType repl_type, bool optional, 
       StringPairSet &alphabet);
    HFSTDLL HfstTransducer replace_transducer
      (HfstTransducer &t, std::string lm, std::string rm, 
       ReplaceType repl_type, StringPairSet &alphabet);
    HFSTDLL HfstTransducer replace_context
      (HfstTransducer &t, std::string m1, std::string m2, 
       StringPairSet &alphabet);
    HFSTDLL HfstTransducer replace_in_context
      (HfstTransducerPair &context, ReplaceType repl_type, 
       HfstTransducer &t, bool optional, StringPairSet &alphabet);

    /* Used by hfst-calculate. */
    HFSTDLL HfstTransducer restriction
      (HfstTransducerPairVector &contexts, HfstTransducer &mapping, 
       StringPairSet &alphabet,        TwolType twol_type, int direction ); 



    // ***** THE PUBLIC INTERFACE *****

    HFSTDLL HfstTransducer two_level_if(HfstTransducerPair &context, 
                                StringPairSet &mappings, 
                                StringPairSet &alphabet);
    
    HFSTDLL HfstTransducer two_level_only_if(HfstTransducerPair &context, 
                                     StringPairSet &mappings, 
                                     StringPairSet &alphabet);

    HFSTDLL HfstTransducer two_level_if_and_only_if(HfstTransducerPair &context, 
                                            StringPairSet &mappings, 
                                            StringPairSet &alphabet);


    HFSTDLL HfstTransducer replace_up(HfstTransducerPair &context, 
                              HfstTransducer &mapping, 
                              bool optional, 
                              StringPairSet &alphabet);

    HFSTDLL HfstTransducer replace_down(HfstTransducerPair &context, 
                                HfstTransducer &mapping, 
                                bool optional, 
                                StringPairSet &alphabet);

    HFSTDLL HfstTransducer replace_down_karttunen(HfstTransducerPair &context, 
                      HfstTransducer &mapping, 
                      bool optional, 
                      StringPairSet &alphabet);

    HFSTDLL HfstTransducer replace_right(HfstTransducerPair &context, 
                                 HfstTransducer &mapping, 
                                 bool optional, 
                                 StringPairSet &alphabet);

    HFSTDLL HfstTransducer replace_left(HfstTransducerPair &context, 
                                HfstTransducer &mapping, 
                                bool optional, 
                                StringPairSet &alphabet);


    HFSTDLL HfstTransducer replace_up(HfstTransducer &mapping, 
                              bool optional, 
                              StringPairSet &alphabet);

    HFSTDLL HfstTransducer replace_down(HfstTransducer &mapping, 
                                bool optional, 
                                StringPairSet &alphabet);

    HFSTDLL HfstTransducer left_replace_up(     HfstTransducer          &mapping,
                                    bool                optional,
                                    StringPairSet       &alphabet);

    HFSTDLL HfstTransducer left_replace_up( HfstTransducerPair  &context,
                                    HfstTransducer      &mapping,
                                    bool                optional,
                                    StringPairSet       &alphabet);
    HFSTDLL HfstTransducer left_replace_down(HfstTransducerPair &context,
                                         HfstTransducer         &mapping,
                                         bool                           optional,
                                         StringPairSet          &alphabet);

    HFSTDLL HfstTransducer left_replace_down_karttunen( HfstTransducerPair      &context,
                                                                                HfstTransducer          &mapping,
                                                                                bool                            optional,
                                                                                StringPairSet           &alphabet);

    HFSTDLL HfstTransducer left_replace_left(HfstTransducerPair &context,
                                         HfstTransducer         &mapping,
                                         bool                           optional,
                                         StringPairSet          &alphabet);

    HFSTDLL HfstTransducer left_replace_right(HfstTransducerPair        &context,
                                          HfstTransducer                &mapping,
                                          bool                          optional,
                                          StringPairSet         &alphabet);





    HFSTDLL HfstTransducer restriction(HfstTransducerPairVector &contexts, 
                               HfstTransducer &mapping, 
                               StringPairSet &alphabet);

    HFSTDLL HfstTransducer coercion(HfstTransducerPairVector &contexts, 
                            HfstTransducer &mapping, 
                            StringPairSet &alphabet);

    HFSTDLL HfstTransducer restriction_and_coercion(HfstTransducerPairVector &contexts,
                                            HfstTransducer &mapping, 
                                            StringPairSet &alphabet);

    HFSTDLL HfstTransducer surface_restriction(HfstTransducerPairVector &contexts, 
                                       HfstTransducer &mapping, 
                                       StringPairSet &alphabet);

    HFSTDLL HfstTransducer surface_coercion(HfstTransducerPairVector &contexts, 
                                    HfstTransducer &mapping, 
                                    StringPairSet &alphabet);

    HFSTDLL HfstTransducer surface_restriction_and_coercion
      (HfstTransducerPairVector &contexts, 
       HfstTransducer &mapping, 
       StringPairSet &alphabet);

    HFSTDLL HfstTransducer deep_restriction(HfstTransducerPairVector &contexts, 
                                    HfstTransducer &mapping, 
                                    StringPairSet &alphabet);

    HFSTDLL HfstTransducer deep_coercion(HfstTransducerPairVector &contexts, 
                                 HfstTransducer &mapping, 
                                 StringPairSet &alphabet);

    HFSTDLL HfstTransducer deep_restriction_and_coercion
      (HfstTransducerPairVector &contexts, 
       HfstTransducer &mapping, 
       StringPairSet &alphabet);
  }

}


// vim: set ft=cpp.doxygen:
#endif // #ifndef _HFST_TRANSDUCER_H_