HfstInputStream.h

Go to the documentation of this file.

// Copyright (c) 2016 University of Helsinki                          
//                                                                    
// This library is free software; you can redistribute it and/or      
// modify it under the terms of the GNU Lesser General Public         
// License as published by the Free Software Foundation; either       
// version 3 of the License, or (at your option) any later version.
// See the file COPYING included with this distribution for more      
// information.

#ifndef _HFST_INPUTSTREAM_H_
#define _HFST_INPUTSTREAM_H_

#if HAVE_CONFIG_H
#  include <config.h>
#endif

#include "HfstDataTypes.h"

#include "hfstdll.h"

namespace hfst
{

  namespace implementations {
#if HAVE_OPENFST
    class LogWeightInputStream;
#if HAVE_OPENFST_LOG
    class TropicalWeightInputStream;
#endif
#endif
#if HAVE_SFST
    class SfstInputStream;
#endif
#if HAVE_FOMA
    class FomaInputStream;
#endif    
#if HAVE_XFSM
    class XfsmInputStream;
#endif
#if HAVE_MY_TRANSDUCER_LIBRARY
    class MyTransducerLibraryInputStream;
#endif
    class HfstOlInputStream;
  }


  class HfstInputStream
  {
  protected:

    union StreamImplementation
    {
#if HAVE_SFST
      hfst::implementations::SfstInputStream * sfst;
#endif
#if HAVE_OPENFST
      hfst::implementations::TropicalWeightInputStream * tropical_ofst;
#if HAVE_OPENFST_LOG
      hfst::implementations::LogWeightInputStream * log_ofst;
#endif
#endif
#if HAVE_FOMA
      hfst::implementations::FomaInputStream * foma;
#endif
#if HAVE_XFSM
      hfst::implementations::XfsmInputStream * xfsm;
#endif

#if HAVE_MY_TRANSDUCER_LIBRARY
      hfst::implementations::MyTransducerLibraryInputStream * 
        my_transducer_library;
#endif

      hfst::implementations::HfstOlInputStream * hfst_ol;
    };

    /* The backend implementation */
    StreamImplementation implementation; 
    /* Implementation type */
    ImplementationType type;             
    /* Name of next transducer, given in the hfst header */
    std::string name;                    
    std::map<std::string,std::string> props;
    /* How many bytes have been already read by the function 
       when processing the hfst header */
    unsigned int bytes_to_skip;          
    /* The name of the file, if stdin, name is "" */
    std::string filename;                
    /* Whether the current transducer has an hfst header */
    bool has_hfst_header;                

    /* A special case where an OpenFst transducer has no symbol tables but an
       SFST alphabet is appended at the end. Should not occur very often, but
       possible when converting old transducers into version 3.0. transducers */
    bool hfst_version_2_weighted_transducer;
 
    /* The stream that the reading operations use when reading the first
       transducer. Then the type of the transducer is not known so there 
       is no backend implementation whose reading functions could be used.
       If input_stream==NULL, the backend implementation is used */
    std::istream * input_stream;

    /* Basic stream operators, work on input_stream (if not NULL) or on 
       the stream implementation. */

    /* Extract one character from the stream */
    char stream_get(); 

    /* Extract one character from the stream and store it in @a c. */
    char &stream_get(char &c);

    /* Extract one short from the stream and store it in @a i. */
    short &stream_get(short &i);

    /* Extract one unsigned short from the stream and store it in @a i. */
    unsigned short &stream_get(unsigned short &i);

    /* Return character c to the stream */
    void stream_unget(char c); 
    /* Whether the stream is at end */
    bool stream_eof(); 
    /* Get a string from the stream */
    std::string stream_getstring(); 
    /* Return the next character in the stream without extracting it */
    char stream_peek();  
    /* The stream implementation ignores n bytes. */
    void ignore(unsigned int n);

    /* The type of a transducer not supported directly by HFST version 3.0 
       but which can occur in conversion functions. */
    enum TransducerType { 
      /* See the above variable. */
      HFST_VERSION_2_WEIGHTED, 
      /* An SFST transducer with no alphabet, not supported. */
      HFST_VERSION_2_UNWEIGHTED_WITHOUT_ALPHABET,
      /* Old header + ordinary SFST transducer. */                
      HFST_VERSION_2_UNWEIGHTED,
      /* An OpenFst transducer, can cause problems if it does not have 
         symbol tables. */ 
      OPENFST_TROPICAL_,
      OPENFST_LOG_,
      /* An SFST transducer. */ 
      SFST_, 
      /* A foma transducer. */ 
      FOMA_, 
      /* An xfsm transducer. */
      XFSM_,
      /* Your transducer type */
      //MY_TRANSDUCER_LIBRARY_,
      /* Transducer type not recognized. */ 
      ERROR_TYPE_ 
    };

    /* Read a transducer from the stream. */
    void read_transducer(HfstTransducer &t);
    /* Type of next transducer in the stream. */
    ImplementationType stream_fst_type();

    // methods used by function stream_fst_type
    TransducerType guess_fst_type(int &bytes_read);
    bool read_hfst_header(int &bytes_read);
    bool read_library_header(int &bytes_read);
    int get_header_size(int &bytes_read);                        
    StringPairVector get_header_data(int header_size);                
    void process_header_data
      (StringPairVector &header_data, bool warnings=false); 
    bool set_implementation_specific_header_data
      (StringPairVector &data, unsigned int index);


    bool read_library_header_old(int &bytes_read);
    ImplementationType get_fst_type_old(int &bytes_read); 

  public:

    HFSTDLL HfstInputStream(void);

    HFSTDLL HfstInputStream(const std::string &filename);

    HFSTDLL ~HfstInputStream(void);

    HFSTDLL void close(void);

    HFSTDLL bool is_eof(void);
    HFSTDLL bool is_bad(void);
    HFSTDLL bool is_good(void);
    
    HFSTDLL ImplementationType get_type(void) const;

    friend class HfstTransducer;
  };


}



#endif