/*

                                   _JNJ`                        
                               .JNMH`                           
                              JMMF`       `;.                   
                            .NMM)          `MN.                 
                            MMM)            (MML                
                           (MMM`             MMML               
                     M  I  D  N  I  G  H  T    C  o  D  E       
                           (NMMF             MHNH               
                            NMML            .MMM                
                             NMML          .NMH                 
                              4MMNL       .#F                   
                               `4HNNL_   `                      
                                   `"""`                        

                          Copyright (C) 2004-2009
       "Ian (Larry) Latter" <ian dot latter at midnightcode dot org>

    Midnight Code is a registered trademark of Ian Latter.

    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, and mirrored at the Midnight Code web
    site; as at version 2 of the License only.
  
    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
    General Public License for more details.
  
    You should have received a copy of the GNU General Public License
    (version 2) along with this program; if not, write to the Free
    Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
    02111-1307 USA, or see http://midnightcode.org/gplv2.txt

*/

/* babel.c :: Midnight Code Babel Compression example program		*/


#include <stdio.h>
#include <stdarg.h>
#include <stdlib.h>
#include <string.h>
#include <fcntl.h>
#include <unistd.h>
#include <sys/stat.h>
#include <openssl/md4.h>
#include <openssl/sha.h>
#include "midnightcode-list.h"



/*
 -- Customisation -------------------------------------------------------------
*/

#define CHUNK_SIZE		3800
#define FILE_RAW		"enwik8"
#define FILE_COMPRESSED		"enwik8.crimsonhexagon@3800"
#define DEBUG			0

/*
                            Compression Reference Chart
               (Assumes that the compressed record consumes 40 bytes)
                  (The compressed record is currently at 38 bytes)
  +--------------------------------------------------------------------------+
  |                           Raw File Chunk Buffer                          |
  +--------------+---------+---------+---------+---------+---------+---------+
  |   Chunk Size |       4 |      40 |     400 |   4,000 |  40,000 | 400,000 |
  +--------------+---------+---------+---------+---------+---------+---------+
  |                             Compression Ratios                           |
  +--------------+---------+---------+---------+---------+---------+---------+
  | Ratio (r:c)  |    1:10 |     1:1 |    10:1 |   100:1 |  1000:1 | 10000:1 |
  | Percentage   |   1000% |    100% |     10% |      1% |    0.1% |   0.01% |
  +--------------+---------+---------+---------+---------+---------+---------+
  |                          Utilisation Data (100MB)                        |
  +--------------+---------+---------+---------+---------+---------+---------+
  | Blocks       |     25m |    2.5m |    250k |     25k |    2.5k |     250 |
  | Block Memory |    1.3G |    130M |     13M |    1.3M |    130K |     13K |
  | Crimson File |      1G |    100M |     10M |      1M |    100K |     10K |
  +--------------+---------+---------+---------+---------+---------+---------+


  Note that the compression success will ultimately be dependant upon the
  collision domain space of the supporting hash functions.

  It would be possible to recursively compress a Crimson Hexagon file (so
  long as the compression ratio remains tighter than 1:1), using Babel to
  deduce high compression ratios with low quality hashes. Additional hashes
  can be added to Babel to sure up larger collision domains, with a 
  diminishing return.

*/
    


/*
 -- Hashing Models ------------------------------------------------------------
*/

#define hash1_length 16
#define hash1_calculate(source, destination, length)		\
  MD4(source, length, destination);
#define hash1_evaluate(hasha, hashb)				\
  memcmp(hasha, hashb, hash1_length)
unsigned char hash1_buffer[hash1_length];
unsigned char hash1_working_buffer[hash1_length];

#define hash2_length 20
#define hash2_calculate(source, destination, length)		\
  SHA1(source, length, destination);
#define hash2_evaluate(hasha, hashb)				\
  memcmp(hasha, hashb, hash2_length)
unsigned char hash2_buffer[hash2_length];
unsigned char hash2_working_buffer[hash2_length];

#define map_length 2
// max == sizeof(unsigned int)
#define map_max 65535



/*
 -- Data Structures -----------------------------------------------------------
*/

struct block_t {
  unsigned int map;
  unsigned char hash1[hash1_length];
  unsigned char hash2[hash2_length];
  int index;
  /*
    To improve decompression performance, the use of 
    per block guidance has been removed.

    int comprised;
    int min_char;
    int max_char;
  */
  midnightcode_list_entry(block_t)
};

midnightcode_list_head(blocks_t, block_t)

struct blocks_t * blocks;



/*
 -- Hash Map Assets -----------------------------------------------------------
*/

struct blocks_t map[65536];
struct blocks_t * map_p;
int map_i;



/*
 -- Constants -----------------------------------------------------------------
*/

#define METADATA		"Crimson Hexagon\0"
#define METADATA_LENGTH		16

enum comprised_t
{
  COMPRISED_BINARY		= 0,
  COMPRISED_CONTROL		= 1,
  COMPRISED_SPECIAL_1		= 2,
  COMPRISED_NUMERIC		= 4,
  COMPRISED_SPECIAL_2		= 8,
  COMPRISED_ALPHA_UC		= 16,
  COMPRISED_SPECIAL_3		= 32,
  COMPRISED_ALPHA_LC		= 64,
  COMPRISED_SPECIAL_4		= 128
};

int block_size_min		= 3;
int block_size_max		= 400000;
unsigned char read_buffer[CHUNK_SIZE];
unsigned char brute_buffer[CHUNK_SIZE];
unsigned int map_buffer;
int last_index			= 0;



/*
 -- Macros --------------------------------------------------------------------
*/

/*
  To improve decompression performance, this high-
  frequency function was authored for run-time
  speed, rather than elegance, and then macro'd.
  This may be further improved with binary math
  operators and perhaps some mask testing.
*/

#define increment(buffer_p, block_size, comprised, min_char, max_char, carry) \
  carry = 1;							\
  for(dynamic_p = buffer_p + block_size - 1;			\
    dynamic_p >= buffer_p; dynamic_p--) {			\
    if(carry) {							\
      if(*dynamic_p + carry > max_char) {			\
        *dynamic_p = min_char;					\
        carry = 1;						\
        continue;						\
      } else {							\
        if(comprised) {						\
          if(*dynamic_p == 31) {				\
            if(comprised & COMPRISED_SPECIAL_1) {		\
              *dynamic_p = 32;					\
              carry = 0;					\
              break;						\
            } else {						\
              if(comprised & COMPRISED_NUMERIC) {		\
                *dynamic_p = 48;				\
                carry = 0;					\
                break;						\
              } else {						\
                if(comprised & COMPRISED_SPECIAL_2) {		\
                  *dynamic_p = 58;				\
                  carry = 0;					\
                  break;					\
                } else {					\
                  if(comprised & COMPRISED_ALPHA_UC) {		\
                    *dynamic_p = 65;				\
                    carry = 0;					\
                    break;					\
                  } else {					\
                    if(comprised & COMPRISED_SPECIAL_3)	 {	\
                      *dynamic_p = 91;				\
                      carry = 0;				\
                      break;					\
                    } else {					\
                      if(comprised & COMPRISED_ALPHA_LC) {	\
                        *dynamic_p = 97;			\
                        carry = 0;				\
                        break;					\
                      } else {					\
                        if(comprised & COMPRISED_SPECIAL_4) {	\
                          *dynamic_p = 123;			\
                          carry = 0;				\
                          break;				\
                        } else {				\
                          *dynamic_p = min_char;		\
                          carry = 1;				\
                          continue;				\
                        }					\
                      }						\
                    }						\
                  }						\
                }						\
              }							\
            }							\
          } else {						\
            if(*dynamic_p == 47) {				\
              if(comprised & COMPRISED_NUMERIC) {		\
                *dynamic_p = 48;				\
                carry = 0;					\
                break;						\
              } else {						\
                if(comprised & COMPRISED_SPECIAL_2) {		\
                  *dynamic_p = 58;				\
                  carry = 0;					\
                  break;					\
                } else {					\
                  if(comprised & COMPRISED_ALPHA_UC) {		\
                    *dynamic_p = 65;				\
                    carry = 0;					\
                    break;					\
                  } else {					\
                    if(comprised & COMPRISED_SPECIAL_3)	 {	\
                      *dynamic_p = 91;				\
                      carry = 0;				\
                      break;					\
                    } else {					\
                      if(comprised & COMPRISED_ALPHA_LC) {	\
                        *dynamic_p = 97;			\
                        carry = 0;				\
                        break;					\
                      } else {					\
                        if(comprised & COMPRISED_SPECIAL_4) {	\
                          *dynamic_p = 123;			\
                          carry = 0;				\
                          break;				\
                        } else {				\
                          *dynamic_p = min_char;		\
                          carry = 1;				\
                          continue;				\
                        }					\
                      }						\
                    }						\
                  }						\
                }						\
              }							\
            } else {						\
              if(*dynamic_p == 57) {				\
                if(comprised & COMPRISED_SPECIAL_2) {		\
                  *dynamic_p = 58;				\
                  carry = 0;					\
                  break;					\
                } else {					\
                  if(comprised & COMPRISED_ALPHA_UC) {		\
                    *dynamic_p = 65;				\
                    carry = 0;					\
                    break;					\
                  } else {					\
                    if(comprised & COMPRISED_SPECIAL_3)	 {	\
                      *dynamic_p = 91;				\
                      carry = 0;				\
                      break;					\
                    } else {					\
                      if(comprised & COMPRISED_ALPHA_LC) {	\
                        *dynamic_p = 97;			\
                        carry = 0;				\
                        break;					\
                      } else {					\
                        if(comprised & COMPRISED_SPECIAL_4) {	\
                          *dynamic_p = 123;			\
                          carry = 0;				\
                          break;				\
                        } else {				\
                          *dynamic_p = min_char;		\
                          carry = 1;				\
                          continue;				\
                        }					\
                      }						\
                    }						\
                  }						\
                }						\
              } else {						\
                if(*dynamic_p == 64) {				\
                  if(comprised & COMPRISED_ALPHA_UC) {		\
                    *dynamic_p = 65;				\
                    carry = 0;					\
                    break;					\
                  } else {					\
                    if(comprised & COMPRISED_SPECIAL_3)	 {	\
                      *dynamic_p = 91;				\
                      carry = 0;				\
                      break;					\
                    } else {					\
                      if(comprised & COMPRISED_ALPHA_LC) {	\
                        *dynamic_p = 97;			\
                        carry = 0;				\
                        break;					\
                      } else {					\
                        if(comprised & COMPRISED_SPECIAL_4) {	\
                          *dynamic_p = 123;			\
                          carry = 0;				\
                          break;				\
                        } else {				\
                          *dynamic_p = min_char;		\
                          carry = 1;				\
                          continue;				\
                        }					\
                      }						\
                    }						\
                  }						\
                } else {					\
                  if(*dynamic_p == 90) {			\
                    if(comprised & COMPRISED_SPECIAL_3)	 {	\
                      *dynamic_p = 91;				\
                      carry = 0;				\
                      break;					\
                    } else {					\
                      if(comprised & COMPRISED_ALPHA_LC) {	\
                        *dynamic_p = 97;			\
                        carry = 0;				\
                        break;					\
                      } else {					\
                        if(comprised & COMPRISED_SPECIAL_4) {	\
                          *dynamic_p = 123;			\
                          carry = 0;				\
                          break;				\
                        } else {				\
                          *dynamic_p = min_char;		\
                          carry = 1;				\
                          continue;				\
                        }					\
                      }						\
                    }						\
                  } else {					\
                    if(*dynamic_p == 96) {			\
                      if(comprised & COMPRISED_ALPHA_LC) {	\
                        *dynamic_p = 97;			\
                        carry = 0;				\
                        break;					\
                      } else {					\
                        if(comprised & COMPRISED_SPECIAL_4) {	\
                          *dynamic_p = 123;			\
                          carry = 0;				\
                          break;				\
                        } else {				\
                          *dynamic_p = min_char;		\
                          carry = 1;				\
                          continue;				\
                        }					\
                      }						\
                    } else {					\
                      if(*dynamic_p == 122) {			\
                        if(comprised & COMPRISED_SPECIAL_4) {	\
                          *dynamic_p = 123;			\
                          carry = 0;				\
                          break;				\
                        } else {				\
                          *dynamic_p = min_char;		\
                          carry = 1;				\
                          continue;				\
                        }					\
                      } else {					\
                        *dynamic_p += carry;			\
                        carry = 0;				\
                        break;					\
                      }						\
                    }						\
                  }						\
                }						\
              }							\
            }							\
          }							\
        } else {						\
          *dynamic_p += carry;					\
          carry = 0;						\
          break;						\
        }							\
      }								\
    }								\
    break;							\
  }

#define increment_reset(buffer_p, block_size, mbuffer, min_char) \
  memset(buffer_p, min_char, block_size);			\
  memcpy(buffer_p, &map_buffer, map_length);



/*
 -- Decoder Functions ---------------------------------------------------------
*/

int
decode_retire_blocks(struct blocks_t * blocks) {
  struct block_t * hold_block;
  struct block_t * block;


  midnightcode_list_for_each_safe(block, hold_block, blocks) {
    midnightcode_list_remove(blocks, block);
    free(block);
    midnightcode_list_for_each_safe_loop(block, hold_block, blocks);
  }

  /*
  file_offset = METADATA_LENGTH + 4;
  if(lseek(fd, file_offset, SEEK_SET) == -1) {
    printf("Error:   Failed to seek to file offset %d.\n", file_offset);
    exit(1);
  }
  */


  return midnightcode_list_count(blocks);
}



int
decode_block(int fd, struct blocks_t * blocks, int block_size,
             unsigned char * buffer_p, unsigned char * raw_buffer,
             int last_pad) {
  struct block_t * block;
  struct block_t * hold_block;
  unsigned char * hash2_p;
  int hash2_calculated;
  int file_offset;


  hash2_calculated = 0;
  midnightcode_list_for_each_safe(block, hold_block, blocks) {
    if(hash1_evaluate(buffer_p, block->hash1) != 0) {
      midnightcode_list_for_each_safe_loop(block, hold_block, blocks);
      continue;
    }
    if(!hash2_calculated) {
      memset(hash2_working_buffer, 0, hash2_length);
      hash2_p = hash2_working_buffer;
      hash2_calculate(raw_buffer, hash2_p, block_size);
    }
    if(hash2_evaluate(hash2_working_buffer, block->hash2) != 0) {
      midnightcode_list_for_each_safe_loop(block, hold_block, blocks);
      continue;
    }
    printf("Notice:  Completed match (removing block %d).\n", block->index);
    file_offset = block->index * block_size;
    if(lseek(fd, file_offset, SEEK_SET) == -1) {
      printf("Error:   Failed to seek to file offset %d.\n", file_offset);
      exit(1);
    }
    if(block->index != (last_index - 1)) {
      if(write(fd, raw_buffer, block_size) != block_size) {
        printf("Error:   Failed to write data block %d.\n", block->index);
        exit(1);
      }
    } else {
      if(write(fd, (raw_buffer + last_pad), (block_size - last_pad))
        != (block_size - last_pad)) {
        printf("Error:   Failed to write last data block %d.\n", block->index);
        exit(1);
      }
    }
    midnightcode_list_remove(blocks, block);
    free(block);
    midnightcode_list_for_each_safe_loop(block, hold_block, blocks);
  }


  return midnightcode_list_count(blocks);
}



int
decode_blocks(int fd_raw, int fd_compressed,
              struct blocks_t * blocks, int block_size,
              int comprised, int min_char, int max_char, int last_pad,
              unsigned int min_map, unsigned int max_map) {
  unsigned char * hash1_working_p;
  unsigned char * dynamic_p;
  int there_is_work_to_do;
  int block_count;
  int carry;


  there_is_work_to_do = 1;
  memset(brute_buffer, min_char, sizeof(brute_buffer));
  hash1_working_p = hash1_working_buffer;
  map_buffer = min_map;
  map_p = &map[map_buffer];

  while(there_is_work_to_do) {
    hash1_calculate(brute_buffer, hash1_working_p, block_size);
    block_count = decode_block(fd_raw, map_p, block_size,
    // block_count = decode_block(fd_raw, blocks, block_size,
      hash1_working_buffer, brute_buffer, last_pad);
    increment(brute_buffer, block_size, comprised, min_char, max_char, carry);
    if(carry)
      return -1;

    while(block_count == 0) {
      map_buffer++;
      if(map_buffer > map_max) {
        there_is_work_to_do = 0;
        break;
      }
      increment_reset(brute_buffer, block_size, map_buffer, min_char)
      decode_retire_blocks(map_p);
      map_p = &map[map_buffer];
      block_count = midnightcode_list_count(map_p);
      
      /*
      decode_retire_blocks(fd_compressed, blocks);
      block_count = decode_read_blocks(fd_compressed, blocks,
        map_buffer, map_buffer);
      if(DEBUG)
        printf("DEBUG: map_buffer=%4.4X, report %d blocks, got %d blocks\n", 
          map_buffer, block_count,
          midnightcode_list_count(blocks));
      */
    }
    if(block_count < 0)
      return -1;
  }


  return 0;
}



int
decode_write_empty(int fd, int block_count, int block_size, int last_pad) {
  int total_write;
  int write_size;
  char * buffer_p;


  total_write = 0;
  write_size = 512;

  if(last_pad) {
    block_count--;
    total_write = block_size - last_pad;
  }
  total_write += block_count * block_size;
  printf("Info:      Empty file is %d bytes, last_pad was %d\n", total_write,
    last_pad);

  if((buffer_p = malloc(write_size)) == NULL)
    return -1;
  while(total_write) {
    if(total_write >= write_size) {
      if(write(fd, buffer_p, write_size) != write_size)
        return -1;
      total_write -= write_size;
    }
    if(total_write > 0 && total_write < write_size) {
      if(write(fd, buffer_p, total_write) != total_write)
        return -1;
      total_write = 0;
    }
  }
  free(buffer_p);


  return 0;
}



int
decode_read_metadata(int fd) {
  char metadata_buffer[METADATA_LENGTH];


  memset(metadata_buffer, 0, sizeof(metadata_buffer));
  if(read(fd, metadata_buffer, METADATA_LENGTH) != METADATA_LENGTH)
    return -1;
  if(memcmp(METADATA, metadata_buffer, METADATA_LENGTH) != 0)
    return 1;


  return 0;
}



int
decode_read_guidance(int fd, int * comprised_p, int * min_char_p,
                     int * max_char_p, int * last_pad_p) {


  *comprised_p = 0;
  *min_char_p = 0;
  *max_char_p = 0;
  *last_pad_p = 0;

  if(read(fd, comprised_p, 1) != 1)
    return -1;
  if(read(fd, min_char_p, 1) != 1)
    return -1;
  if(read(fd, max_char_p, 1) != 1)
    return -1;
  if(read(fd, last_pad_p, 4) != 4)
    return -1;


  return 0;
}



int
decode_read_block(int fd, struct block_t * block,
                  unsigned int min_map, unsigned int max_map) {
  int max_read;
  unsigned char map_place[map_length];
  int skip;


  skip = 0;
  memset(block, 0, sizeof(struct block_t));

  memset(map_place, 0, map_length);
  if((max_read = read(fd, &map_place, map_length)) != map_length) {
    if(max_read == 0)
      return 0;
    return -1;
  }
  memcpy(&(block->map), &map_place, map_length);

  if(min_map == max_map) {
    if(block->map != min_map)
      skip = 1;
  } else {
    if(block->map < min_map || block->map > max_map)
      skip = 1;
  }
  if(skip) {
    if(lseek(fd, (hash1_length + hash2_length), SEEK_CUR) == -1) {
      printf("Error:   Failed to seek to end of block in data file.\n");
      exit(1);
    }
    return 2;
  }
  if(DEBUG)
    printf("DEBUG: ????? [%4.4X] == [%4.4X]\n", block->map, min_map);
  if((max_read = read(fd, block->hash1, hash1_length)) != hash1_length) {
    if(max_read == 0)
      return 0;
    return -1;
  }
  if((max_read = read(fd, block->hash2, hash2_length)) != hash2_length) {
    if(max_read == 0)
      return 0;
    return -1;
  }


  return 1;
}



int
decode_read_blocks(int fd, struct blocks_t * blocks,
                   unsigned int min_map, unsigned int max_map) {
  struct block_t * block;
  int max_read;
  int block_index;


  block_index = 0;
  while(1) {
    if((block = malloc(sizeof(struct block_t))) == NULL)
      return -1;
    if((max_read = decode_read_block(fd, block, min_map, max_map)) < 0) {
      free(block);
      return -1;
    }
    if(max_read == 0) {
      free(block);
      break;
    }
    if(max_read == 2) {
      free(block);
      block_index++;
      continue;
    }
    block->index = block_index;
    block_index++;
    map_p = &map[block->map];
    midnightcode_list_append(map_p, block);
    // midnightcode_list_append(blocks, block);
  }
  last_index = block_index;


  return midnightcode_list_count(blocks);
}



int
decode_read_file(int fd, struct blocks_t * blocks, int * comprised_p,
                 int * min_char_p, int * max_char_p, int * last_pad_p,
                 unsigned int min_map, unsigned int max_map) {


  if(decode_read_metadata(fd) != 0) {
    printf("Error:   Metadata failed for data file.\n");
    return -1;
  }
  if(decode_read_guidance(fd, comprised_p, min_char_p, max_char_p,
    last_pad_p) < 0)
    return -1;
  if(decode_read_blocks(fd, blocks, min_map, max_map) < 0)
    return -1;


  return 0;
}



/*
 -- Encoder Functions ---------------------------------------------------------
*/

int
encode_block(struct block_t * block, unsigned char * raw_buffer,
             int block_size) {
  int block_c;
  unsigned char * hash1_p;
  int hash1_c;
  unsigned char * hash2_p;
  int hash2_c;


  memset(&block->map, 0, map_length);
  memcpy(&block->map, raw_buffer, map_length);

  memset(block->hash1, 0, hash1_length);
  hash1_p = block->hash1;
  hash1_calculate(raw_buffer, hash1_p, block_size);

  memset(block->hash2, 0, hash2_length);
  hash2_p = block->hash2;
  hash2_calculate(raw_buffer, hash2_p, block_size);

  /*
    To improve decompression performance, the use of 
    per block guidance has been removed.

    encode_minmax(raw_buffer, block_size, 0,
      &block->comprised, &block->min_char, &block->max_char);
  */

  if(DEBUG) {
    printf("Debug:   Data set [");
    for(block_c = 0; block_c < block_size; block_c++) {
      printf("%c", raw_buffer[block_c]);
    }
    printf("]\n");
    printf("Debug:   Hash1=[");
    for(hash1_c = 0; hash1_c < hash1_length; hash1_c++) {
      printf("%2.2X", block->hash1[hash1_c]);
    }
    printf("]\n");
    printf("Debug:   Hash2=[");
    for(hash2_c = 0; hash2_c < hash2_length; hash2_c++) {
      printf("%2.2X", block->hash2[hash2_c]);
    }
    printf("]\n");
  }


  return 0;
}



int
encode_blocks(int fd, struct blocks_t * blocks,
              unsigned char * buffer_p, int block_size,
              int min_char, int * last_pad_p) {
  struct block_t * block;
  char read_test[1];
  char * read_test_p;
  int max_read;


  while((max_read = read(fd, buffer_p, block_size)) == block_size) {
    if((block = malloc(sizeof(struct block_t))) == NULL)
      return -1;
    encode_block(block, buffer_p, block_size);
    /*
      To improve compression performance, encode_block_write
      could be called directly from here, rather than building
      the file-sized link-list in memory.  This would reduce
      compression memory consumption to a single hash pair.
    */
    midnightcode_list_append(blocks, block);
  }
  if(max_read < 0)
    return -1;
  if(max_read == 0)
    *last_pad_p = 0;
  if(max_read > 0) {
    read_test_p = read_test;
    if(read(fd, read_test_p, 1) != 0)
      return -1;
    if((block = malloc(sizeof(struct block_t))) == NULL)
      return -1;
    *last_pad_p = block_size - max_read;
    memmove((buffer_p + *last_pad_p), buffer_p, max_read);
    memset(buffer_p, min_char, *last_pad_p);
    encode_block(block, buffer_p, block_size);
    midnightcode_list_append(blocks, block);
  }


  return midnightcode_list_count(blocks);
}



int
encode_minmax(unsigned char * buffer_p, int block_size, int continued,
              int * comprised_p, int * min_char_p, int * max_char_p) {
  int buffer_offset;
  int binary;
  unsigned char * dynamic_p;


  binary = 0;
  if(!continued) {
    *comprised_p = 0;
    *min_char_p = 255;
    *max_char_p = 0;
  }

  for(buffer_offset = block_size - 1; buffer_offset >= 0; buffer_offset--) {
    dynamic_p = buffer_p + buffer_offset;
    if(*dynamic_p > *max_char_p)
      *max_char_p = *dynamic_p;
    if(*dynamic_p < *min_char_p)
      *min_char_p = *dynamic_p;
    if(!binary) {
      if(*dynamic_p > 127) {
        binary = 1;
        *comprised_p = 0;
      } else {
        if(*dynamic_p <= 31) {
          *comprised_p |= COMPRISED_CONTROL;
        } else {
          if(*dynamic_p >= 32 && *dynamic_p <= 47) {
            *comprised_p |= COMPRISED_SPECIAL_1;
          } else {
            if(*dynamic_p >= 48 && *dynamic_p <= 57) {
              *comprised_p |= COMPRISED_NUMERIC;
            } else {
              if(*dynamic_p >= 58 && *dynamic_p <= 64) {
                *comprised_p |= COMPRISED_SPECIAL_2;
              } else {
                if(*dynamic_p >= 65 && *dynamic_p <= 90) {
                  *comprised_p |= COMPRISED_ALPHA_UC;
                } else {
                  if(*dynamic_p >= 91 && *dynamic_p <= 96) {
                    *comprised_p |= COMPRISED_SPECIAL_3;
                  } else {
                    if(*dynamic_p >= 97 && *dynamic_p <= 122) {
                      *comprised_p |= COMPRISED_ALPHA_LC;
                    } else {
                      if(*dynamic_p >= 123 && *dynamic_p <= 127) {
                        *comprised_p |= COMPRISED_SPECIAL_4;
                      }
                    }
                  }
                }
              }
            }
          }
        }
      }
    }
  }

  if(!continued) {
    if(DEBUG) {
      printf("Debug:   Comprised=[%d]\n", *comprised_p);
      printf("Debug:   min_char=[%d] max_char=[%d]\n",*min_char_p,*max_char_p);
    }
  }


  return binary;
}



int
encode_minmax_file(int fd, unsigned char * buffer_p, int block_size,
                   int * comprised_p, int * min_char_p, int * max_char_p) {
  int max_read;
  int binary;


  binary = 0;
  *comprised_p = 0;
  *min_char_p = 255;
  *max_char_p = 0;

  memset(buffer_p, 0, block_size);
  while((max_read = read(fd, buffer_p, block_size)) > 0) {
    binary |= encode_minmax(read_buffer, block_size, 1,
      comprised_p, min_char_p, max_char_p);
    memset(buffer_p, 0, block_size);
  }
  if(binary)
    *comprised_p = 0;

  if(lseek(fd, 0, SEEK_SET) == -1) {
    printf("Error:   Failed to seek to start of data file %s.\n",
      FILE_COMPRESSED);
    exit(1);
  }


  return 0;
}



int
encode_write_metadata(int fd) {



  if(write(fd, METADATA, METADATA_LENGTH) != METADATA_LENGTH)
    return -1;


  return 0;
}



int
encode_write_guidance(int fd, unsigned char * comprised_p,
                      unsigned char * min_char_p, unsigned char * max_char_p,
                      unsigned char * last_pad_p) {



  if(write(fd, comprised_p, 1) != 1)
    return -1;
  if(write(fd, min_char_p, 1) != 1)
    return -1;
  if(write(fd, max_char_p, 1) != 1)
    return -1;
  if(write(fd, last_pad_p, 4) != 4)
    return -1;


  return 0;
}



int
encode_write_block(int fd, struct block_t * block) {


  if(write(fd, &block->map, map_length) != map_length)
    return -1;
  if(write(fd, block->hash1, hash1_length) != hash1_length)
    return -1;
  if(write(fd, block->hash2, hash2_length) != hash2_length)
    return -1;


  return 0;
}



int
encode_write_blocks(int fd, struct blocks_t * blocks) {
  struct block_t * block;
  struct block_t * hold_block;


  midnightcode_list_for_each_safe(block, hold_block, blocks) {
    if(encode_write_block(fd, block) < 0)
      return -1;
    midnightcode_list_remove(blocks, block);
    free(block);
    midnightcode_list_for_each_safe_loop(block, hold_block, blocks);
  }


  return 0;
}



int
encode_write_file(int fd, struct blocks_t * blocks, int * comprised_p,
                  int * min_char_p, int * max_char_p, int * last_pad_p) {


  if(encode_write_metadata(fd) < 0)
    return -1;
  if(encode_write_guidance(fd, (unsigned char *)comprised_p,
    (unsigned char *)min_char_p, (unsigned char *)max_char_p,
    (unsigned char *)last_pad_p) < 0)
    return -1;
  if(encode_write_blocks(fd, blocks) < 0)
    return -1;


  return 0;
}



/*
 -- Core Function -------------------------------------------------------------
*/

int
main(int argc, char **argv) {
  int fd_raw;
  int fd_compressed;
  int decoder;
  int comprised_file;
  int min_char_file;
  int max_char_file;
  int last_pad_file;
  unsigned int min_map;
  unsigned int max_map;
  int ret_i;
  int block_size = CHUNK_SIZE;
  int block_count;
  void * void_p;

  midnightcode_list_init(block_head, blocks_t);
  void_p = (void *)&block_head;
  blocks = void_p;


  for(map_i = 0; map_i <= 65535; map_i++)
    memset(&(map[map_i]), 0, sizeof(struct blocks_t));

  ret_i = 0;
  decoder = 0;
  min_map = 0x0000;
  max_map = 0xFFFF;
  memset(read_buffer, 0, sizeof(read_buffer));
  memset(brute_buffer, 0, sizeof(brute_buffer));
  memset(&map_buffer, 0, sizeof(map_buffer));

  printf("Notice:  Validating program configuration state.\n");
  printf("Info:      Chunk size is %d.\n", block_size);
  if(block_size < block_size_min) {
    printf("Error:   Chunk size %d is less than the minimum %d.\n",
      block_size, block_size_min);
    exit(1);
  }
  if(block_size > block_size_max) {
    printf("Error:   Chunk size %d is greater than the maximum %d.\n",
      block_size, block_size_max);
    exit(1);
  }
  if(map_length > block_size) {
    printf("Error:   Map length %d is greater than Chunk size %d.\n",
      map_length, block_size);
    exit(1);
  }

  printf("Notice:  Operating in [Encoder] mode.\n");
  printf("Notice:  Opening raw data file: %s\n", FILE_RAW);
  if((fd_raw = open(FILE_RAW, O_RDONLY)) != -1) {

    printf("Notice:  Creating compressed data file: %s\n", FILE_COMPRESSED);
    if(access(FILE_COMPRESSED, R_OK) == 0) {
      /*
        To improve decompression functionality, an existing
        but incomplete compressed data file could be resumed
        if a decompression state file was also maintained by
        the decompressor.
      */
      close(fd_raw);
      printf("Error:   Data file %s already exists.\n", FILE_COMPRESSED);
      exit(1);
    }
    if((fd_compressed = open(FILE_COMPRESSED, O_WRONLY|O_CREAT, S_IRUSR))
      == -1) {
      close(fd_raw);
      printf("Error:   Unable to create data file %s.\n", FILE_COMPRESSED);
      exit(1);
    }

    printf("Notice:  Analysing raw data for decompressor guidance.\n");
    encode_minmax_file(fd_raw, read_buffer, block_size,
      &comprised_file, &min_char_file, &max_char_file);
    printf("Info:      Comprised=%d, MinChar=%d, MaxChar=%d\n",
      comprised_file, min_char_file, max_char_file);

    printf("Notice:  Encoding raw data as hash block(s).\n");
    block_count = encode_blocks(fd_raw, blocks, read_buffer, block_size,
      min_char_file, &last_pad_file);
    printf("Info:      LastPad=%d\n", last_pad_file);

    printf("Notice:  Writing %d compressed blocks to data file.\n",
      block_count);
    if(encode_write_file(fd_compressed, blocks,
      &comprised_file, &min_char_file, &max_char_file, &last_pad_file) < 0) {
      close(fd_compressed);
      close(fd_raw);
      printf("Error:   Failed to write compressed data blocks.\n");
      exit(1);
    }

    printf("Notice:  Compressor completed.\n");
    close(fd_compressed);
    close(fd_raw);
    exit(0);

  } else {

    printf("Warning: Failed to open raw data file.\n");
    printf("Notice:  Switched to [Decoder] mode.\n");

    printf("Notice:  Opening compressed data file: %s\n", FILE_COMPRESSED);
    if((fd_compressed = open(FILE_COMPRESSED, O_RDONLY)) == -1) {
      printf("Error:   Failed to open %s data file.\n", FILE_COMPRESSED);
      exit(1);
    }

    printf("Notice:  Creating raw data file: %s\n", FILE_RAW);
    if(access(FILE_RAW, R_OK) == 0) {
      printf("Error:   Data file %s already exists.\n", FILE_RAW);
      close(fd_compressed);
      exit(1);
    }
    if((fd_raw = open(FILE_RAW, O_WRONLY|O_CREAT, S_IRUSR)) == -1) {
      printf("Error:   Unable to create data file %s.\n", FILE_RAW);
      close(fd_compressed);
      exit(1);
    }

    printf("Notice:  Reading blocks from compressed data file.\n");
    if(decode_read_file(fd_compressed, blocks, &comprised_file,
      &min_char_file, &max_char_file, &last_pad_file, min_map, max_map) < 0) {
      printf("Error:   Unable to import blocks from data file.\n");
      close(fd_compressed);
      close(fd_raw);
      exit(1);
    }
    printf("Info:      Comprised=%d, MinChar=%d, MaxChar=%d, LastPad=%d\n",
      comprised_file, min_char_file, max_char_file, last_pad_file);
    for(map_i = 0; map_i <= 65535; map_i++) {
      map_p = &map[map_i];
      if(DEBUG)
        printf("Info:      hash map bucket [%d] total = %d\n", map_i,
          midnightcode_list_count(map_p));
    }

    printf("Notice:  Padding file with (~ %d x %d bytes of) null data.\n",
      last_index, block_size);
    if(decode_write_empty(fd_raw, last_index, block_size, last_pad_file) < 0) {
      printf("Error:   Unable to pad out raw data file.\n");
      close(fd_compressed);
      close(fd_raw);
      exit(1);
    }

    printf("Notice:  Searching for the Crimson Hexagon ..\n");
    decode_blocks(fd_raw, fd_compressed, blocks, block_size, comprised_file,
      min_char_file, max_char_file, last_pad_file, min_map, max_map);


    printf("Notice:  Decompressor completed.\n");
    close(fd_compressed);
    close(fd_raw);
    exit(0);
  }


  exit(0);
}



/*
 -- Notes ---------------------------------------------------------------------
*/

/*
  Note;


    - It was a design decision that functions in this program deliberately
      do not validate input.  The security of this proof-of-concept program
      was not factored into its scope.


    - The Crimson Hexagon file format;

      It consists of one Header (23) and one or more Block (38) structures.

        Header (23);
          +---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+
          | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | A | B | C | D | E | F |
          +---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+
          |                         Meta Data (16)                        |
          +---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+
          |        Guidance (7)       |
          +---+---+---+---+---+---+---+

          Guidance consists of;
            Comprised (1) | Min Char (1) | Max Char (1) | Last Pad (4) |
      
        Block (38);
          +---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+
          | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | A | B | C | D | E | F |
          +---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+
          |  Map  |                          Hash 1 (16)                  
          +---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+
                  |                                      Hash 2 (20) 
          +---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+
                                  |
          +---+---+---+---+---+---+

      Note that the last block of raw data is padded left.


    - Apparently this model (walking through every version of every file of
      size n) was considered in a number of documents early in the 20th
      Century. Jorge Luis Borges' 1941 thought piece "The Library of Babel"
      is the most identifiable;

        [...]

        Second: The orthographical symbols are twenty-five in
        number. This finding made it possible, three hundred
        years ago, to formulate a general theory of the
        Library and solve satisfactorily the problem which no
        conjecture had deciphered: the formless and chaotic
        nature of almost all the books. One which my father
        saw in a hexagon on circuit fifteen ninety-four was
        made up of the letters MCV, perversely repeated from
        the first line to the last. Another (very much
        consulted in this area) is a mere labyrinth of
        letters, but the next-to-last page says; Oh time thy
        pyramids. This much is already known: for every
        sensible line of straightforward statement, there
        are leagues of senseless cacophonies, verbal jumbles
        and incoherences. (I know of an uncouth region whose
        librarians repudiate the vain and superstitious
        custom of finding a meaning in books and equate it
        with that of finding a meaning in dreams or in the
        chaotic lines of one's palm ... They admit that the
        inventors of this writing imitated the twenty-five
        natural symbols, but maintain that this application
        is accidental and that the books signify nothing in
        themselves. This dictum, we shall see, is not
        entirely fallacious.)

        For a long time it was believed that these
        impenetrable books corresponded to past or remote
        languages. It is true that the most ancient men, the
        first librarians, used a language quite different
        from the one we now speak; it is true that a few
        miles to the right the tongue is dialectical and that
        ninety floors farther up, it is incomprehensible. All
        this, I repeat, is true, but four hundred and ten
        pages of inalterable MCV's cannot correspond to any
        language, no matter how dialectical or rudimentary it
        may be. Some insinuated that each letter could
        influence the following one and that the value of MCV
        in the third line of page 71 was not the one the same
        series may have in another position on another page,
        but this vague thesis did not prevail. Others thought
        of cryptographs; generally, this conjecture has been
        accepted, though not in the sense in which it was
        formulated by its originators.

        Five hundred years ago, the chief of an upper hexagon
        came upon a book as confusing as the others, but
        which had nearly two pages of homogeneous lines. He
        showed his find to a wandering decoder who told him
        the lines were written in Portuguese; others said
        they were Yiddish. Within a century, the language was
        established: a Samoyedic Lithuanian dialect of
        Guarani, with classical Arabian inflections. The
        content was also deciphered: some notions of
        combinative analysis, illustrated with examples of
        variations with unlimited repetition. These examples
        made it possible for a librarian of genius to
        discover the fundamental law of the Library. This
        thinker observed that all the books, no matter how
        diverse they might be, are made up of the same
        elements: the space, the period, the comma, the
        twenty-two letters of the alphabet. He also alleged a
        fact which travelers have confirmed: In the vast
        Library there are no two identical books. From these
        two incontrovertible premises he deduced that the
        Library is total and that its shelves register all
        the possible combinations of the twenty-odd
        orthographical symbols (a number which, though
        extremely vast, is not infinite): Everything: the
        minutely detailed history of the future, the
        archangels' autobiographies, the faithful catalogues
        of the Library, thousands and thousands of false
        catalogues, the demonstration of the fallacy of those
        catalogues, the demonstration of the fallacy of the
        true catalogue, the Gnostic gospel of Basilides, the
        commentary on that gospel, the commentary on the
        commentary on that gospel, the true story of your
        death, the translation of every book in all
        languages, the interpolations of every book in all
        books.

        [...]
        
        Counter to general opinion, I venture to suppose that
        the consequences of the Purifiers' depredations have
        been exaggerated by the horror these fanatics
        produced. They were urged on by the delirium of
        trying to reach the books in the Crimson Hexagon:
        books whose format is smaller than usual, all-
        powerful, illustrated and magical. 

        [...]

      From;
        http://jubal.westnet.com/hyperdiscordia/library_of_babel.html

      See also;
        http://en.wikipedia.org/wiki/The_Library_of_Babel


    - The naming conventions for this program were adapted to the literary
      work after it was found that the two shared this model.


    - This program takes no parameters specifically to oblige the Hutter
      Prize objective; parametisation would be straight forward, and the
      Crimson Hexagon Header could be adapted to include the block/chunk
      size to make the whole program more portable.


    - Regarding the Hutter Prize, satisfaction of the rules and the
      objective to achieve the highest compression ratios increases with
      higher values for CHUNK_SIZE (approaching 400,000 bytes).  Note that
      while this significantly increases time it remains operable within
      the rule constraints;

        Rules
         * Publish a program (self-extracting archive)
           archive8.exe of size S.
         * If run, it produces a 108 byte file data8
           that is identical to enwik8.
         * Programs must be Windows or Linux (x86 32bit)
           executables.
         * Programs must run without input from other
           sources (files, network, dictionaries, etc.)
           under Windows or Linux without additional
           installations. Use of standard libraries as
           for file I/O are allowed.
         * Programs should run in less than 10 hours on
           a 2GHz Pentium 4 with 1GB RAM and 10GB free
           HD for temporary files.

      From;
        http://prize.hutter1.net/hrules.htm


    - Outside of the Hutter Prize (which is performance limited)
      performance can be increased through a number of orders of magnitudes
      via an assortment of mechanisms.

      For your reference, an AMD Sempron 2200+ (single core) can produce
      the MD4 (Hash1) hashes at a rate of about 700,000 hashes per second.

       - Today;

          - Process Distribution;

            The brute-force exploration of the block (or key-space) could
            be very easily split off into independant brute-force efforts,
            allowing work-load to be distributed amongst thousands of
            CPU cores, or thousands of Computers.  As it stands today, 
            the program is ripe for distribution to 65,536 processing 
            nodes (split along the hash map).

          - Processor Architecture;

            The underlying processor architecture, regardless of its speed,
            has an impact upon the performance of the hashing process;
              http://cr.yp.to/talks/2008.06.05/slides.pdf

          - MD5 Accelerators

            Consider replacing the underlying Hash1 function with MD5;
            MD5(); = 128 bits (16 bytes)
              http://www.openssl.org/docs/crypto/md5.html
            Soekris VPN1401 (PCI with Hi/FN 7955) = 360 Mega-bits-per-second
              http://www.soekris.com/vpn1401.htm
            -----------------------------------------------------------
              360,000,000 / 128 ~= 2.8 million MD5's per second.
                2 x PCI Card ~= 5 million MD5's per second.
                3 x PCI Card ~= 8 million MD5's per second.

          - DES-CBC Accelerators

            Consider replacing the underlying Hash1 function with DES-CBC;
            des_cbc_cksum(); = 64 bits (8 bytes)
              http://www.openssl.org/docs/crypto/des.html
            SafeNet SafeXcel EIP-16b = 3040 Mega-bits-per-second
              http://www.safenet-inc.com/library/emb/
              SafeNet_Product_Brief_SafeXcel_DES_3DES_Accelerators.pdf
            -----------------------------------------------------------
              3,040,000,000 / 64 = 47.5 million DES-CBC's per second.
                2 x PCI Card ~= 100 million DES-CBC's per second.
                3 x PCI Card ~= 150 million DES-CBC's per second.

       - Tomorrow;

          - Ray Kurzweil's analysis suggests that human-brain capable
            hardware will be available sub US$1,000 by about 2021;
              http://www.ted.com/index.php/talks/
              ray_kurzweil_on_how_technology_will_transform_us.html

            If all else fails, then it is only a matter of time.


    - A hash decompressor, such as this program, could be integrated into
      the Internet infrastructure in a number of ways.  However, the most
      practical today would be through the dynamic (per-user) P2P 
      infrastructure, and the DHT protocol.


*/
