//
// LDASTools frameCPP - A library implementing the LIGO/Virgo frame
// specification
//
// Copyright (C) 2018 California Institute of Technology
//
// LDASTools frameCPP is free software; you may redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 (GPLv2) of the
// License or at your discretion, any later version.
//
// LDASTools frameCPP is distributed in the hope that it will be useful, but
// without any warranty or even the implied warranty of merchantability
// or fitness for a particular purpose. See the GNU General Public
// License (GPLv2) for more details.
//
// Neither the names of the California Institute of Technology (Caltech),
// The Massachusetts Institute of Technology (M.I.T), The Laser
// Interferometer Gravitational-Wave Observatory (LIGO), nor the names
// of its contributors may be used to endorse or promote products derived
// from this software without specific prior written permission.
//
// You should have received a copy of the licensing terms for this
// software included in the file LICENSE located in the top-level
// directory of this package. If you did not, you can view a copy at
// http://dcc.ligo.org/M1500244/LICENSE
//

//-----------------------------------------------------------------------
// This program is used to try out ideas for compressing frames files.
// It is not intended for general use, but instead be a record of
// ideas that have been tried.
// If ideas from here pan out, they should be formalized into the
// frame specification and then implemented within the FrameCPP library.
//-----------------------------------------------------------------------

#include <zlib.h>

#include <algorithm>
#include <fstream>
#include <map>
#include <memory>
#include <set>

#include <boost/shared_ptr.hpp>

#include "ldastoolsal/MemChecker.hh"
#include "ldastoolsal/CommandLineOptions.hh"
#include "ldastoolsal/fstream.hh"
#include "ldastoolsal/regex.hh"
#include "ldastoolsal/regexmatch.hh"
#include "ldastoolsal/types.hh"
#include "ldastoolsal/unordered_map.hh"

#include "framecpp/Common/FrameBuffer.hh"
#include "framecpp/Common/FrameStream.hh"
#include "framecpp/Common/MemoryBuffer.hh"
#include "framecpp/Common/StreamRefInterface.hh"

#include "framecpp/FrameCPP.hh"

using LDASTools::AL::MemChecker;
using LDASTools::AL::unordered_map;

using FrameCPP::Common::MemoryBuffer;
using FrameCPP::Common::OFrameStream;
using FrameCPP::Common::OStream;

typedef LDASTools::AL::CommandLineOptions CommandLineOptions;
typedef CommandLineOptions::Option        Option;
typedef CommandLineOptions::OptionSet     OptionSet;

typedef FrameCPP::Common::FrameBuffer< LDASTools::AL::filebuf >
    frame_buffer_type;
typedef boost::shared_ptr< FrameCPP::Common::StreamRefInterface >
    stream_ref_type;

#define DEFAULT_CHANNEL_PATTERN_RULE 5
#define SPLIT_SUBCHAN 1

static const char* DEFAULT_CHANNEL_PATTERN =
#if DEFAULT_CHANNEL_PATTERN_RULE == 1
    "^([^-]*-)(.*)([.](min|max|mean|n|rms))$"
#elif DEFAULT_CHANNEL_PATTERN_RULE == 2
    "^([^_]*_)(.*)([.](min|max|mean|n|rms))$"
#elif DEFAULT_CHANNEL_PATTERN_RULE == 3
    "^(.*)(_[^_]*)([.](min|max|mean|n|rms))$"
#elif DEFAULT_CHANNEL_PATTERN_RULE == 4
    "^([^-]*-)(.*)(_[^_]*)([.](min|max|mean|n|rms))$"
#elif DEFAULT_CHANNEL_PATTERN_RULE == 5
    "^([[:upper:]][[:digit:]]:[^-]*-)(([^_.]*)(_[^_.]+)*)(([.].*)|)$"
#endif
    ;

static const char* DEFAULT_SENTENCE_PATTERN =
#if 0
  "^([[:alnum:][,][:][!][?]]+)([[:space:]]+[[:alnum:][,][:][!][?]]+)*$"
#else
    "^([[:alnum:]!?,:]+)([[:space:]]+[[:alnum:]!?,:]+)*$"
#endif
    ;

//-----------------------------------------------------------------------
// Forward declaration
//-----------------------------------------------------------------------

class IStream;

typedef IStream stream_type;

static void depart( int ExitCode );

//'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
// class CommandLine - Definition
//'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
class CommandLine : protected CommandLineOptions
{
public:
    typedef INT_4U block_size_type;

    CommandLine( int ArgC, char** ArgV );

    block_size_type BlockSize( ) const;

    void Eval( );

    bool MemoryMappedIO( ) const;

    void Usage( int ExitValue ) const;

    using CommandLineOptions::empty;
    using CommandLineOptions::Pop;
    using CommandLineOptions::size;

private:
    enum compression_type
    {
        COMPRESSION_BLOCK,
        COMPRESSION_FR_STRUCTS,
        COMPRESSION_STREAM
    };

    enum mode_type
    {
        MODE_COMPRESS,
        MODE_CREATE_DICTIONARY
    };

    enum option_type
    {
        OPT_HELP,
        OPT_BLOCK_SIZE,
        OPT_COMPRESSION_MODE,
        OPT_CREATE_DICTIONARY,
        OPT_DICTIONARY_FILE,
        OPT_LEVEL,
        OPT_SEGMENT_SIZE
    };

    OptionSet        options;
    block_size_type  block_size;
    bool             memory_mapped_io;
    compression_type compression;
    std::string      dictionary_file;
    int              level;
    mode_type        mode;
    block_size_type  segment_size;

    void compress_fr_structs( );
};

//'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
// class Compress - Definition
//'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
class Compress
{
public:
    Compress( );
    ~Compress( );

    void operator( )( stream_type& Stream );

protected:
    size_t buffer_size_total;
    size_t compressed_size_total;

    virtual void compress( const std::string& Buffer ) = 0;
};

//'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
// class ZLibWithDictionary - Definition
//'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
class ZLibWithDictionary : public Compress
{
public:
    static const int DEFAULT_LEVEL = 1;

    ZLibWithDictionary( const std::string& DictionaryFilename,
                        int                Level = DEFAULT_LEVEL );

protected:
    typedef boost::scoped_array< unsigned char > out_type;

    static const size_t CHUNK = ( 64 * 1024 );
    static const int    MAX_DICTIONARY_SIZE = ( 32 * 1024 );

    std::string                          dictionary;
    int                                  level;
    boost::scoped_array< unsigned char > out;
    z_stream                             stream;

    virtual void open_stream( );

    virtual void close_stream( );

    virtual void compress_stream( const std::string& Buffer, int Flag );
};

//'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
// class CompressGZipWithDictionary - Definition
//'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
class CompressGZipWithDictionary : public ZLibWithDictionary
{
public:
    CompressGZipWithDictionary( const std::string& DictionaryFilename,
                                int                Level = DEFAULT_LEVEL );

protected:
    virtual void compress( const std::string& Buffer );
};

//'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
// class CompressStreamGZipWithDictionary - Definition
//'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
class CompressStreamGZipWithDictionary : public ZLibWithDictionary
{
public:
    static const int DEFAULT_LEVEL = 1;

    CompressStreamGZipWithDictionary( const std::string& DictionaryFilename,
                                      int Level = DEFAULT_LEVEL );
    ~CompressStreamGZipWithDictionary( );

protected:
    virtual void compress( const std::string& Buffer );

    virtual void close_stream( );
};

//'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
// class CompressGZipWithDictionary - Definition
//'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
class CompressZLibBlocking : public ZLibWithDictionary
{
public:
    static const size_t DEFAULT_BLOCK_SIZE = ( 16 * 1024 * 1024 );

    CompressZLibBlocking( size_t             BlockSize,
                          const std::string& DictionaryFilename,
                          int                Level = DEFAULT_LEVEL );

    CompressZLibBlocking( const std::string& DictionaryFilename,
                          int                Level = DEFAULT_LEVEL );

    virtual ~CompressZLibBlocking( );

protected:
    virtual void compress( const std::string& Buffer );

    virtual void compress_stream( const std::string& Buffer, int Flag );

private:
    const size_t        block_size;
    size_t              count;
    size_t              current_block_size;
    mutable std::string verify_source_buffer;
    mutable std::string verify_compressed_buffer;

    void verify( ) const;
};

//'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
// DictionaryAnalyser
//'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
class DictionaryAnalyser
{
public:
    typedef unordered_map< std::string, int >        stats_type;
    typedef std::map< int, std::set< std::string > > ordered_type;

    DictionaryAnalyser( size_t MinimumWordSize = 2 );

    void Add( const std::string& Value );
    void Analyse( );
    void Dump( std::ostream& Stream ) const;

private:
    const size_t MINIMUM_WORD_SIZE;

    stats_type   stats;
    ordered_type ordered;
};

//'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
// class IStream - Definition
//'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
class IStream : public FrameCPP::Common::IFrameStream
{
public:
    IStream( buffer_type* Stream );

    void ReadBlock( std::string& Block, size_t Size );

    void ReadFrSH( stream_ref_type Ref );

    void Rewind( stream_ref_type Ref );

    using FrameCPP::Common::IFrameStream::readStreamRef;
};

//-----------------------------------------------------------------------
// Main
//-----------------------------------------------------------------------
int
main( int ArgC, char** ArgV )
{
    MemChecker::Trigger gc_trigger( true );
    int                 exit_status( 0 );

    try
    {
        CommandLine cl( ArgC, ArgV );

        FrameCPP::Initialize( );

        cl.Eval( );
    }
    catch ( std::exception& e )
    {
        std::cerr << "ABORT: Caught exception: " << e.what( ) << std::endl;
        depart( 1 );
    }

    return exit_status;
}

//=======================================================================
// CommandLine
//=======================================================================
CommandLine::CommandLine( int ArgC, char** ArgV )
    : CommandLineOptions( ArgC, ArgV ), block_size( 1024 ),
      compression( COMPRESSION_STREAM ), level( 1 ), mode( MODE_COMPRESS ),
      segment_size( 1024 )
{
    typedef unordered_map< std::string, compression_type >
        compression_container_type;

    compression_container_type compression_container;

    INT_4U multiplier = 1024;
    INT_4U segment_multiplier = 1024;

    compression_container[ "block" ] = COMPRESSION_BLOCK;
    compression_container[ "stream" ] = COMPRESSION_STREAM;
    compression_container[ "structs" ] = COMPRESSION_FR_STRUCTS;

    //---------------------------------------------------------------------
    // Description information
    //---------------------------------------------------------------------
    options.Synopsis( "[options] <file> [<file> ...]" );

    options.Summary( "This command is used to test proposed frame"
                     " compression schemes." );

    //---------------------------------------------------------------------
    // Declaration of options
    //---------------------------------------------------------------------
    std::ostringstream desc;

    //'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
    desc.str( "" );
    desc << "Specify the size in bytes of the input and output buffer."
            " If the number is followed by the letter 'b', 'k' or 'M',"
            " then the buffer size is multiplied by 1, 1024 or"
            " 1048576 respectively. (Default: "
         << BlockSize( ) << ( ( multiplier == 1024 ) ? "k" : "M" ) << " )";

    options.Add( Option( OPT_BLOCK_SIZE,
                         "block-size",
                         Option::ARG_REQUIRED,
                         desc.str( ),
                         "size" ) );

    //'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
    desc.str( "" );
    desc << "This specifies the compression mode to be used.";

    options.Add( Option( OPT_COMPRESSION_MODE,
                         "compression-mode",
                         Option::ARG_REQUIRED,
                         desc.str( ),
                         "mode" ) );

    //'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
    desc.str( "" );
    desc << "This modifies the mode of the program."
            " In this mode, standard input is read and a dictionary"
            " is written to standard out.";

    options.Add( Option( OPT_CREATE_DICTIONARY,
                         "create-dictionary",
                         Option::ARG_NONE,
                         desc.str( ) ) );

    //'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
    desc.str( "" );
    desc << "Specify the name of the dictionary file."
            " When creating a dictionary, this file is used to store"
            " the dictionary."
            " When testing compression modes, this file is used to seed"
            " the dictionary if applicable to the compression method"
            " being tested.";

    options.Add( Option( OPT_DICTIONARY_FILE,
                         "dictionary-file",
                         Option::ARG_REQUIRED,
                         desc.str( ),
                         "filename" ) );

    //'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
    desc.str( "" );
    desc << "Specify the numeric level of compression for methods that support "
            "it.";

    options.Add( Option(
        OPT_LEVEL, "level", Option::ARG_REQUIRED, desc.str( ), "level" ) );

    //'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''
    desc.str( "" );
    desc << "Specify the size in bytes of how to segment the input buffer."
            " If the number is followed by the letter 'b', 'k' or 'M',"
            " then the buffer size is multiplied by 1, 1024 or"
            " 1048576 respectively."
            " (Default: "
         << segment_size << ( ( segment_multiplier == 1024 ) ? "k" : "M" )
         << " )";

    options.Add( Option( OPT_SEGMENT_SIZE,
                         "segment-size",
                         Option::ARG_REQUIRED,
                         desc.str( ),
                         "size" ) );

    //---------------------------------------------------------------------
    // Parse the options specified on the command line
    //---------------------------------------------------------------------
    try
    {
        std::string arg_name;
        std::string arg_value;
        bool        parsing = true;

        while ( parsing )
        {
            const int cmd_id( Parse( options, arg_name, arg_value ) );

            switch ( cmd_id )
            {
            case CommandLineOptions::OPT_END_OF_OPTIONS:
                parsing = false;
                break;
            case OPT_HELP:
            {
                Usage( 0 );
            }
            break;
            case OPT_BLOCK_SIZE:
            {
                if ( arg_value.length( ) >= 1 )
                {
                    switch ( arg_value[ arg_value.length( ) - 1 ] )
                    {
                    case 'b':
                        multiplier = 1;
                        break;
                    case 'k':
                        multiplier = 1024;
                        break;
                    case 'M':
                        multiplier = 1024 * 1024;
                        break;
                    default:
                        break;
                    }
                }

                std::istringstream num( arg_value.substr(
                    0,
                    ( multiplier == 1 ) ? std::string::npos
                                        : arg_value.length( ) - 1 ) );
                num >> block_size;
            }
            break;
            case OPT_COMPRESSION_MODE:
            {
                compression_container_type::const_iterator c =
                    compression_container.find( arg_value );

                if ( c != compression_container.end( ) )
                {
                    compression = c->second;
                }
                else
                {
                    std::cerr
                        << "WARN:"
                        << " Skipping unknown compression mode: " << arg_value
                        << std::endl;
                }
            }
            break;
            case OPT_CREATE_DICTIONARY:
                mode = MODE_CREATE_DICTIONARY;
                break;
            case OPT_DICTIONARY_FILE:
                dictionary_file = arg_value;
                break;
            case OPT_LEVEL:
            {
                std::istringstream num( arg_value );

                num >> level;
            }
            break;
            case OPT_SEGMENT_SIZE:
            {
                if ( arg_value.length( ) >= 1 )
                {
                    switch ( arg_value[ arg_value.length( ) - 1 ] )
                    {
                    case 'b':
                        segment_multiplier = 1;
                        break;
                    case 'k':
                        segment_multiplier = 1024;
                        break;
                    case 'M':
                        segment_multiplier = 1024 * 1024;
                        break;
                    default:
                        break;
                    }
                }

                std::istringstream num( arg_value.substr(
                    0,
                    ( segment_multiplier == 1 ) ? std::string::npos
                                                : arg_value.length( ) - 1 ) );
                num >> segment_size;
            }
            break;
            }
        }
    }
    catch ( ... )
    {
    }

    block_size *= multiplier;
    segment_size *= segment_multiplier;
}

inline CommandLine::block_size_type
CommandLine::BlockSize( ) const
{
    return block_size;
}

void
CommandLine::Eval( )
{
    switch ( mode )
    {
    case MODE_COMPRESS:
    {
        //-----------------------------------------------------------------
        // Ensure there is at least one file to compress
        //-----------------------------------------------------------------
        if ( empty( ) )
        {
            Usage( 1 );
        }

        //-----------------------------------------------------------------
        // Temporary buffer
        //-----------------------------------------------------------------
        boost::scoped_array< CHAR > buffer( new CHAR[ BlockSize( ) ] );

        //-----------------------------------------------------------------
        // Loop over the files
        //-----------------------------------------------------------------
        while ( empty( ) == false )
        {
            //---------------------------------------------------------------
            // Open the stream
            //---------------------------------------------------------------
            std::string                          filename( Pop( ) );
            std::unique_ptr< frame_buffer_type > ifile(
                new frame_buffer_type( std::ios::in ) );

            ifile->UseMemoryMappedIO( MemoryMappedIO( ) );
            ifile->open( filename.c_str( ), std::ios::in | std::ios::binary );

            IStream ifstream( ifile.release( ) );
            switch ( compression )
            {
            case COMPRESSION_BLOCK:
            {
                CompressZLibBlocking compressor(
                    segment_size, dictionary_file, level );

                compressor( ifstream );
            }
            break;
            case COMPRESSION_FR_STRUCTS:
            {
                CompressGZipWithDictionary compressor( dictionary_file, level );

                compressor( ifstream );
            }
            break;
            case COMPRESSION_STREAM:
            {
                CompressStreamGZipWithDictionary compressor( dictionary_file,
                                                             level );

                compressor( ifstream );
            }
            break;
            default:
                std::cerr << "WARNING:"
                          << " Unsupported compression mode: " << compression
                          << " at: " << __FILE__ << ":" << __LINE__
                          << std::endl;
                break;
            }
        }
    }
    break;
    case MODE_CREATE_DICTIONARY:
    {
        Regex              channel_pattern( DEFAULT_CHANNEL_PATTERN );
        Regex              sentence_pattern( DEFAULT_SENTENCE_PATTERN );
        RegexMatch         channel_match( 15 );
        RegexMatch         sentence_match( 15 );
        std::string        line;
        DictionaryAnalyser dictionary;

        while ( std::getline( std::cin, line ) )
        {
            //---------------------------------------------------------------
            // Parse the line using regex
            //---------------------------------------------------------------
            bool is_channel( false );
            bool is_sentence( false );

            if ( ( ( is_channel = channel_match.match(
                         channel_pattern, line.c_str( ) ) ) == false ) &&
                 ( ( is_sentence = sentence_match.match(
                         sentence_pattern, line.c_str( ) ) ) == false ) )
            {
                continue;
            }

            if ( is_channel )
            {
                dictionary.Add( channel_match.getSubString( 1 ) );
                dictionary.Add( channel_match.getSubString( 5 ) );

#if SPLIT_SUBCHAN
                std::string subchan = channel_match.getSubString( 2 );
                for ( size_t cur = 0, last = 0, eos = subchan.size( );
                      cur != eos; )
                {
                    while ( ( last != eos ) && ( subchan[ last ] != '_' ) )
                    {
                        last++;
                    }
                    dictionary.Add( subchan.substr( cur, last - cur ) );
                    cur = last;
                    while ( ( last != eos ) && ( subchan[ last ] == '_' ) )
                    {
                        ++last;
                    }
                }
#else
                dictionary.Add( channel_match.getSubString( 2 ) );
#endif
            }
            else if ( is_sentence )
            {
                for ( size_t cur = 0, last = 0, eos = line.size( );
                      cur != eos; )
                {
                    while ( ( last != eos ) && ( !ispunct( line[ last ] ) ) &&
                            ( !isspace( line[ last ] ) ) )
                    {
                        last++;
                    }
                    dictionary.Add( line.substr( cur, last - cur ) );
                    while ( ( last != eos ) &&
                            ( ( ispunct( line[ last ] ) ) ||
                              ( isspace( line[ last ] ) ) ) )
                    {
                        last++;
                    }
                    cur = last;
                }
            }
            else
            {
                std::cerr << "WARNING: Skipped: " << line << std::endl;
            }
        }
        dictionary.Analyse( );
        dictionary.Dump( std::cout );
    }
    break;
    }
}

inline bool
CommandLine::MemoryMappedIO( ) const
{
    return memory_mapped_io;
}

void
CommandLine::Usage( int ExitValue ) const
{
    std::cout << "Usage: " << ProgramName( ) << options << std::endl;
    depart( ExitValue );
}

//=======================================================================
// Compress
//=======================================================================
Compress::Compress( ) : buffer_size_total( 0 ), compressed_size_total( 0 )
{
}

Compress::~Compress( )
{
    std::cerr << "INFO:"
              << " Buffer Total: " << buffer_size_total
              << " Cmprss Total: " << compressed_size_total << " Percent: "
              << ( ( double( compressed_size_total ) /
                     double( buffer_size_total ) ) *
                   100. )
              << std::endl;
}

void
Compress::operator( )( stream_type& Stream )
{
    try
    {
        stream_ref_type sref;
        std::string     block;

        while ( 1 )
        {
            sref = Stream.readStreamRef( );
            if ( !sref )
            {
                break;
            }
            if ( sref->GetClass( ) == 1 )
            {
                //---------------------------------------------------------------
                // Need to satisfy the lookup dictionary by seeding the mapping
                // of ids beyond 2.
                //---------------------------------------------------------------
                Stream.ReadFrSH( sref );
                Stream.Rewind( sref );
            }
            //-----------------------------------------------------------------
            // Read into a buffer to try compression algorithms
            //-----------------------------------------------------------------
            Stream.ReadBlock( block, sref->GetLength( ) - sref->SizeOf( ) );
#if 0
      std::cerr << "DEBUG:"
		<< " GetLength( ): " << sref->GetLength( )
		<< " SizeOf(): " << sref->SizeOf( )
		<< " block.size( ): " << block.size( )
		<< std::endl
	;
#endif /* 0 */
            compress( block );
        }
    }
    catch ( const std::exception& e )
    {
        std::cerr << "FATAL: Exception: " << e.what( ) << std::endl;
    }
    catch ( ... )
    {
        std::cerr << "FATAL: Exception" << std::endl;
    }
}

//=======================================================================
// CompressGZipWithDictionary
//=======================================================================
CompressGZipWithDictionary::CompressGZipWithDictionary(
    const std::string& DictionaryFilename, int Level )
    : ZLibWithDictionary( DictionaryFilename, Level )
{
}

void
CompressGZipWithDictionary::compress( const std::string& Buffer )
{
    open_stream( );
    compress_stream( Buffer, Z_FINISH );
    close_stream( );
}

//=======================================================================
//=======================================================================
CompressStreamGZipWithDictionary::CompressStreamGZipWithDictionary(
    const std::string& DictionaryFilename, int Level )
    : ZLibWithDictionary( DictionaryFilename, Level )
{
    //---------------------------------------------------------------------
    // Setup compression buffer
    //---------------------------------------------------------------------
    open_stream( );
}

CompressStreamGZipWithDictionary::~CompressStreamGZipWithDictionary( )
{
    close_stream( );
}

void
CompressStreamGZipWithDictionary::close_stream( )
{
    std::string empty;

    compress_stream( empty, Z_FINISH );
    ZLibWithDictionary::close_stream( );
}

void
CompressStreamGZipWithDictionary::compress( const std::string& Buffer )
{
    compress_stream( Buffer, Z_NO_FLUSH );
}

//=======================================================================
// DictionaryAnalyser
//=======================================================================

DictionaryAnalyser::DictionaryAnalyser( size_t MinimumWordSize )
    : MINIMUM_WORD_SIZE( MinimumWordSize )
{
}

void
DictionaryAnalyser::Add( const std::string& Value )
{
    if ( Value.size( ) >= MINIMUM_WORD_SIZE )
    {
        stats[ Value ] += 1;
    }
}

void
DictionaryAnalyser::Analyse( )
{
    for ( stats_type::const_iterator cur = stats.begin( ), last = stats.end( );
          cur != last;
          ++cur )
    {
        if ( cur->second > 1 )
        {
            //-----------------------------------------------------------------
            // It appears more than once, so it make sence to try and put
            //   it into the dictionary.
            //-----------------------------------------------------------------
#if 0
      ordered[ cur->second * cur->first.length( ) ].insert( cur->first );
#else
            ordered[ cur->second ].insert( cur->first );
#endif
        }
    }
}

void
DictionaryAnalyser::Dump( std::ostream& Stream ) const
{
    for ( ordered_type::const_iterator cur = ordered.begin( ),
                                       last = ordered.end( );
          cur != last;
          ++cur )
    {
        for ( std::set< std::string >::const_iterator
                  set_cur = cur->second.begin( ),
                  set_last = cur->second.end( );
              set_cur != set_last;
              ++set_cur )
        {
            Stream << *set_cur;
        }
    }
}

//=======================================================================
// IStream
//=======================================================================
IStream::IStream( buffer_type* Stream )
    : FrameCPP::Common::IFrameStream( Stream )
{
}

void
IStream::ReadBlock( std::string& Block, size_t Size )
{
    char   tmp[ 1024 ];
    size_t r( 0 );

#if 0
  std::cerr << "DEBUG:"
	    << " Started at: " << tellg( )
	    << std::endl
    ;
#endif /* 0 */
    Block.clear( );
    while ( Size > 0 )
    {
        r = std::min( Size, sizeof( tmp ) );

        read( tmp, r );
        Size -= r;
        Block.append( tmp, r );
    }
#if 0
  std::cerr << "DEBUG:"
	    << " Ended at: " << tellg( )
	    << std::endl
    ;
#endif /* 0 */
}

void
IStream::ReadFrSH( stream_ref_type Ref )
{
    (void)readObject( Ref.get( ) );
}

void
IStream::Rewind( stream_ref_type Ref )
{
    //---------------------------------------------------------------------
    // Rewind to the beginning of the object
    //---------------------------------------------------------------------
    seekg( -( Ref->GetLength( ) - Ref->SizeOf( ) ), std::ios_base::cur );
}

//=======================================================================
// CompressZLibBlocking
//=======================================================================
CompressZLibBlocking::CompressZLibBlocking(
    const std::string& DictionaryFilename, int Level )
    : ZLibWithDictionary( DictionaryFilename, Level ),
      block_size( DEFAULT_BLOCK_SIZE ), count( 0 ), current_block_size( 0 )
{
    open_stream( );
}

CompressZLibBlocking::CompressZLibBlocking(
    size_t BlockSize, const std::string& DictionaryFilename, int Level )
    : ZLibWithDictionary( DictionaryFilename, Level ), block_size( BlockSize ),
      count( 0 ), current_block_size( 0 )
{
    open_stream( );
}

CompressZLibBlocking::~CompressZLibBlocking( )
{
    std::string empty;

    compress_stream( empty, Z_FINISH );
    close_stream( );
    if ( verify_source_buffer.size( ) > 0 )
    {
        ++count;
        verify( );
    }
    std::cerr << "INFO:"
              << " Number of Blocks: " << count << std::endl;
}

void
CompressZLibBlocking::compress( const std::string& Buffer )
{
    size_t remaining( Buffer.size( ) );
    size_t pos( 0 );

    while ( remaining )
    {
        if ( ( current_block_size + remaining ) < block_size )
        {
            //-----------------------------------------------------------------
            // This is the case where the information remaining in the buffer
            // fits into the current block of data being compressed
            //-----------------------------------------------------------------
            if ( pos )
            {
                compress_stream( Buffer.substr( pos ), Z_NO_FLUSH );
            }
            else
            {
                compress_stream( Buffer, Z_NO_FLUSH );
            }
            current_block_size += remaining;
            remaining = 0;
        }
        else
        {
            //-----------------------------------------------------------------
            // Handle the case where only a portion of the buffer can be
            //   placed into the buffer being compressed
            //-----------------------------------------------------------------
            size_t len( block_size - current_block_size );

            compress_stream( Buffer.substr( pos, len ), Z_FULL_FLUSH );
            deflateReset( &stream );
            current_block_size = 0;
            ++count;
            remaining -= len;
            pos += len;
            //-----------------------------------------------------------------
            // Verify that the buffer just deflated can be inflated
            //-----------------------------------------------------------------
            verify( );
        }
    }
}

void
CompressZLibBlocking::verify( ) const
{
    typedef unordered_map< int, std::string > error_type;

    static error_type error_codes;

    bool success = true;
    if ( error_codes.size( ) == 0 )
    {
        error_codes[ Z_NEED_DICT ] = "Z_NEED_DICT";
        error_codes[ Z_DATA_ERROR ] = "Z_DATA_ERROR";
        error_codes[ Z_MEM_ERROR ] = "Z_MEM_ERROR";
    }
    static const size_t VERIFY_OVERFLOW = 512;
    z_stream            vstream;
    size_t   outsize( verify_source_buffer.size( ) + VERIFY_OVERFLOW );
    out_type b( new out_type::element_type[ outsize ] );
    int      ret;

    std::cerr << "DEBUG:"
              << " Verifing buffer of size: " << verify_source_buffer.size( )
              << " compressed size: " << verify_compressed_buffer.size( )
              << std::endl;
    vstream.zalloc = Z_NULL;
    vstream.zfree = Z_NULL;
    vstream.opaque = Z_NULL;
    vstream.avail_in = verify_compressed_buffer.size( );
    vstream.next_in = (Bytef*)( verify_compressed_buffer.c_str( ) );
    inflateInit( &vstream );
    vstream.avail_out = outsize;
    vstream.next_out = (Bytef*)( b.get( ) );
    ret = inflate( &vstream, Z_NO_FLUSH );
    switch ( ret )
    {
    case Z_NEED_DICT:
    case Z_DATA_ERROR:
    case Z_MEM_ERROR:
        std::cerr << "FAIL:"
                  << " Unable to decompress block " << ( count - 1 )
                  << " (Error: " << error_codes[ ret ] << ")" << std::endl;
        std::cerr << "INFO:"
                  << " Bytes decompressed: "
                  << ( outsize - VERIFY_OVERFLOW - vstream.avail_out ) << " of "
                  << ( outsize - VERIFY_OVERFLOW ) << std::endl;
        success = false;
        break;
    }
    if ( success )
    {
        if ( vstream.avail_out != VERIFY_OVERFLOW )
        {
            std::cerr << "FAIL:"
                      << " Buffer inflated is of an incorrect size"
                      << " for buffer " << ( count - 1 ) << std::endl;
            success = false;
        }
    }
    if ( success )
    {
        std::string expanded( (const char*)( b.get( ) ),
                              outsize - VERIFY_OVERFLOW );
        if ( verify_source_buffer.compare( expanded ) != 0 )
        {
            std::cerr << "FAIL:"
                      << " Buffer inflated differs from original buffer"
                      << " for buffer " << ( count - 1 ) << std::endl;
            success = false;
        }
    }
    if ( success )
    {
        std::cerr << "PASS:"
                  << " Buffer inflated properly"
                  << " for buffer " << ( count - 1 ) << std::endl;
    }
    //---------------------------------------------------------------------
    // Prepare for the next round
    //---------------------------------------------------------------------
    inflateEnd( &vstream );
    verify_source_buffer.clear( );
    verify_compressed_buffer.clear( );
}

void
CompressZLibBlocking::compress_stream( const std::string& Buffer, int Flag )
{
    verify_source_buffer += Buffer;
    buffer_size_total += Buffer.size( );

    stream.avail_in = Buffer.size( );
    stream.next_in = (Bytef*)( Buffer.c_str( ) );
    do
    {
        stream.avail_out = CHUNK;
        stream.next_out = (Bytef*)( out.get( ) );
        deflate( &stream, Flag );
        compressed_size_total += ( CHUNK - stream.avail_out );
        verify_compressed_buffer.append( (const char*)( out.get( ) ),
                                         CHUNK - stream.avail_out );
    } while ( stream.avail_out == 0 );
}

//=======================================================================
// class ZLibWithDictionary
//=======================================================================
ZLibWithDictionary::ZLibWithDictionary( const std::string& DictionaryFilename,
                                        int                Level )
    : level( Level )
{
    if ( DictionaryFilename.size( ) > 0 )
    {
        dictionary.resize( 0 );
        std::ifstream source( DictionaryFilename.c_str( ) );
        if ( source.good( ) )
        {
            //-----------------------------------------------------------------
            // Calculate the size of the file
            //-----------------------------------------------------------------
            source.seekg( 0, std::ios_base::end );
            std::streampos s( source.tellg( ) );
            std::streampos bufsize(
                std::min( s, std::streampos( MAX_DICTIONARY_SIZE ) ) );
#if 0
      boost::scoped_array< char >	buf( new char[ bufsize ] );
#endif /* 0 */
            //-----------------------------------------------------------------
            // Position ourselves at the start of the relevant part of the file
            // and then read the dictionary
            //-------------------------------------------------------------------
            source.seekg( -bufsize, std::ios_base::end );
#if 1
            dictionary.resize( bufsize );
            source.read( const_cast< char* >( dictionary.c_str( ) ),

                         bufsize );
#else
            source.read( buf.get( ), bufsize );
            dictionary.assign( buf.get( ), bufsize );
#endif
            source.close( );
        } // if source.good
    } // if DictionaryFilename.size( ) > 0
    out.reset( new out_type::element_type[ CHUNK ] );
}

void
ZLibWithDictionary::open_stream( )
{
    stream.zalloc = Z_NULL;
    stream.zfree = Z_NULL;
    stream.opaque = Z_NULL;

    deflateInit( &stream, level );
    if ( dictionary.size( ) > 0 )
    {
        deflateSetDictionary( &stream,
                              (const Bytef*)( dictionary.c_str( ) ),
                              dictionary.size( ) );
    }
}

void
ZLibWithDictionary::close_stream( )
{
    deflateEnd( &stream );
}

void
ZLibWithDictionary::compress_stream( const std::string& Buffer, int Flag )
{
    buffer_size_total += Buffer.size( );

    stream.avail_in = Buffer.size( );
    stream.next_in = (Bytef*)( Buffer.c_str( ) );
    do
    {
        stream.avail_out = CHUNK;
        stream.next_out = (Bytef*)( out.get( ) );
        deflate( &stream, Flag );
        compressed_size_total += ( CHUNK - stream.avail_out );
    } while ( stream.avail_out == 0 );
}

//=======================================================================
// Local functions
//=======================================================================

inline static void
depart( int ExitCode )
{
    exit( ExitCode );
}
