Slim numerical data compression  1.0
bitstream.h
Go to the documentation of this file.
1 // -*- mode: c++; -*-
2 
5 
6 // Copyright (C) 2008, 2009 Joseph Fowler
7 //
8 // This file is part of slim, a compression package for science data.
9 //
10 // Slim is free software: you can redistribute it and/or modify
11 // it under the terms of the GNU General Public License as published by
12 // the Free Software Foundation, either version 3 of the License, or
13 // (at your option) any later version.
14 //
15 // Slim is distributed in the hope that it will be useful,
16 // but WITHOUT ANY WARRANTY; without even the implied warranty of
17 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 // GNU General Public License for more details.
19 //
20 // You should have received a copy of the GNU General Public License
21 // along with slim. If not, see <http://www.gnu.org/licenses/>.
22 
23 #ifndef SLIM_BITSTREAM_H
24 #define SLIM_BITSTREAM_H
25 
26 #include <stdint.h>
27 #include <cstdio>
28 
29 #ifdef HAVE_LIBZZIP
30 #include <zzip/zzip.h>
31 #endif
32 
33 #ifdef HAVE_LIBLZ4
34 #include <lz4frame.h>
35 #endif
36 
37 #include "bit_constants.h"
38 
39 using namespace std;
40 
41 class bitstream {
42 public:
43  bitstream();
44  bitstream(const char *filename,
45  int buffersize=DEFAULT_IOBUFFER_SIZE);
46  virtual ~bitstream();
47 
48  virtual void close() = 0;
49  virtual bool is_open() const = 0;
50  virtual void setupstream();
51  virtual void windup()=0;
52  virtual int get_bytes_used();
53  int get_bitptr();
54  virtual void print() const = 0;
55 
56 protected:
57  static const int Bits_per_word = 8*sizeof(Word_t);
58 
59  size_t bufsize;
60  size_t buf_used;
61  Byte_t *buffer_base;
62  Byte_t *beyondbuffer;
63  union {
64  Byte_t *Bptr;
65  Word_t *Dptr;
66  } buffptr;
67  int bitptr;
68 
71 public:
72  enum {DEFAULT_IOBUFFER_SIZE=1024*1024};
73  enum {MAX_BITSTREAM_BUFSIZE=16*1024*1024};
74 };
75 
76 
77 
78 class obitstream : public bitstream {
79 private:
80  // No private attributes.
81 
82 protected:
83  FILE *fp;
84 
85 public:
86  obitstream(FILE *file, int buffersize=DEFAULT_IOBUFFER_SIZE);
87  obitstream(const char *filename,
88  int buffersize=DEFAULT_IOBUFFER_SIZE);
89  ~obitstream();
90 
91  void writebits(uint32_t data, int nbits);
92  void writestring(const char *str, bool write_trailing_null=false);
93  template <typename T> void writeword(const T data);
94  void write_unary(unsigned int value);
95  virtual void print() const;
96  virtual void close();
97  virtual bool is_open() const;
98  void windup();
99  void flush(bool flush_trailing_bits);
100 };
101 
102 
103 
104 
105 class ibitstream : public bitstream {
106 private:
107  // No private attributes (unless debugging).
108 #ifdef DEBUG_READBITS
109  int c1,c2, c3, c4, c5, c6;
110  void print_debug();
111 #endif
112 
113 protected:
114 #ifdef HAVE_LIBZZIP
115  ZZIP_FILE *zfp;
116 #endif
117  FILE *fp;
118 #ifdef HAVE_LIBLZ4
119  LZ4F_decompressionContext_t lz4_ctx;
120  Byte_t *lz4_buffer;
121  bool using_lz4;
122 #endif
123 
124 public:
125  ibitstream(FILE *file, int buffersize=DEFAULT_IOBUFFER_SIZE);
126 #ifdef HAVE_LIBZZIP
127  ibitstream(ZZIP_FILE *file, int buffersize=DEFAULT_IOBUFFER_SIZE);
128 #endif
129  ibitstream(const char *filename,
130  int buffersize=DEFAULT_IOBUFFER_SIZE);
131  ibitstream(int fd, int buffersize=DEFAULT_IOBUFFER_SIZE);
132  ~ibitstream();
133 
134  virtual void close();
135  virtual bool is_open() const;
136  void setupstream();
137  void windup();
138  virtual void print() const;
139  virtual int get_bytes_used();
140  Word_t readbits(int nbits);
141  int32_t readbits_int(int nbits);
142  Word_t read_unary();
143  int readstring(char *s, int count=-1);
144  //int get_bits_used() { return bitptr + Bits_per_word*buf_used;}
145 
146 private:
147  void next_word();
148  int fill();
149 
150  Word_t partial_word;
151  int partial_word_bitptr;
152 };
153 
154 
155 
157 // Inline functions
159 
166 static inline unsigned int bit_size(int32_t i) {
167  if (i < 0)
168  i = (-i)-1; // Convert negative int to non-negatives of same size.
169 
170  if (i>lowestNset32bits[15]) {
171  if (i>lowestNset32bits[23]) {
172  if (i>lowestNset32bits[27]) {
173  if (i>lowestNset32bits[29]) {
174  if (i>lowestNset32bits[30]) {
175  return 32;
176  } else {
177  return 31;
178  }
179  } else {
180  if (i>lowestNset32bits[28]) {
181  return 30;
182  } else {
183  return 29;
184  }
185  }
186  } else {
187  if (i>lowestNset32bits[25]) {
188  if (i>lowestNset32bits[26]) {
189  return 28;
190  } else {
191  return 27;
192  }
193  } else {
194  if (i>lowestNset32bits[24]) {
195  return 26;
196  } else {
197  return 25;
198  }
199  }
200  }
201  } else {
202  if (i>lowestNset32bits[19]) {
203  if (i>lowestNset32bits[21]) {
204  if (i>lowestNset32bits[22]) {
205  return 24;
206  } else {
207  return 23;
208  }
209  } else {
210  if (i>lowestNset32bits[20]) {
211  return 22;
212  } else {
213  return 21;
214  }
215  }
216  } else {
217  if (i>lowestNset32bits[17]) {
218  if (i>lowestNset32bits[18]) {
219  return 20;
220  } else {
221  return 19;
222  }
223  } else {
224  if (i>lowestNset32bits[16]) {
225  return 18;
226  } else {
227  return 17;
228  }
229  }
230  }
231  }
232  } else {
233  if (i>lowestNset32bits[7]) {
234  if (i>lowestNset32bits[11]) {
235  if (i>lowestNset32bits[13]) {
236  if (i>lowestNset32bits[14]) {
237  return 16;
238  } else {
239  return 15;
240  }
241  } else {
242  if (i>lowestNset32bits[12]) {
243  return 14;
244  } else {
245  return 13;
246  }
247  }
248  } else {
249  if (i>lowestNset32bits[9]) {
250  if (i>lowestNset32bits[10]) {
251  return 12;
252  } else {
253  return 11;
254  }
255  } else {
256  if (i>lowestNset32bits[8]) {
257  return 10;
258  } else {
259  return 9;
260  }
261  }
262  }
263  } else {
264  if (i>lowestNset32bits[3]) {
265  if (i>lowestNset32bits[5]) {
266  if (i>lowestNset32bits[6]) {
267  return 8;
268  } else {
269  return 7;
270  }
271  } else {
272  if (i>lowestNset32bits[4]) {
273  return 6;
274  } else {
275  return 5;
276  }
277  }
278  } else {
279  if (i>lowestNset32bits[1]) {
280  if (i>lowestNset32bits[2]) {
281  return 4;
282  } else {
283  return 3;
284  }
285  } else {
286  if (i>lowestNset32bits[0]) {
287  return 2;
288  } else {
289  return 1;
290  }
291  }
292  }
293  }
294  }
295 }
296 
297 
298 
303 static inline unsigned int bit_size(unsigned int u) {
304  for (int bs=1; bs<=32; bs++)
305  if (u == (u&lowestNset32bits[bs]))
306  return bs;
307  throw "Bit size (unsigned int) fails!";
308 }
309 
310 
311 #endif // #ifdef SLIM_BITSTREAM_H
ibitstream::readbits_int
int32_t readbits_int(int nbits)
Read data from the buffer as (signed) ints.
Definition: bitstream.cpp:607
bitstream::beyondbuffer
Byte_t * beyondbuffer
Pointer just beyond buffer (convenience).
Definition: bitstream.h:62
obitstream::close
virtual void close()
Close the output stream by flushing and closing the FILE *.
Definition: bitstream.cpp:211
bitstream::Bits_per_word
static const int Bits_per_word
Bits per buffer word.
Definition: bitstream.h:57
bitstream
Definition: bitstream.h:41
ibitstream::get_bytes_used
virtual int get_bytes_used()
Return the number of bytes used so far in this stream.
Definition: bitstream.cpp:509
obitstream::fp
FILE * fp
The I/O stream.
Definition: bitstream.h:83
bitstream::bufsize
size_t bufsize
Size of I/O buffer (bytes)
Definition: bitstream.h:59
obitstream::writeword
void writeword(const T data)
Write all the bits of a word.
Definition: bitstream.cpp:284
ibitstream::read_unary
Word_t read_unary()
Read a single unary-coded value.
Definition: bitstream.cpp:622
bitstream::Dptr
Word_t * Dptr
Pointer to the current word (as Word_t *).
Definition: bitstream.h:65
bitstream::bitptr
int bitptr
Pointer to the current bits.
Definition: bitstream.h:67
ibitstream::readbits
Word_t readbits(int nbits)
Read data from the buffer as unsigned ints.
Definition: bitstream.cpp:556
bitstream::setupstream
virtual void setupstream()
Allocate a buffer and set up all pointers.
Definition: bitstream.cpp:141
ibitstream::setupstream
void setupstream()
Allocate a buffer and set up all pointers, then fill buffer.
Definition: bitstream.cpp:517
obitstream::flush
void flush(bool flush_trailing_bits)
Flush the write buffer and reset for more data.
Definition: bitstream.cpp:351
ibitstream
Definition: bitstream.h:105
bitstream::buffer_base
Byte_t * buffer_base
Pointer to the buffer.
Definition: bitstream.h:61
obitstream::is_open
virtual bool is_open() const
Is the IO file closed?
Definition: bitstream.cpp:124
bitstream::Bptr
Byte_t * Bptr
Pointer to the current word (as Byte_t *).
Definition: bitstream.h:64
ibitstream::is_open
virtual bool is_open() const
Is the IO file closed?
Definition: bitstream.cpp:130
obitstream::obitstream
obitstream(FILE *file, int buffersize=DEFAULT_IOBUFFER_SIZE)
Start obitstream using a FILE ptr to an open file.
Definition: bitstream.cpp:182
obitstream::writebits
void writebits(uint32_t data, int nbits)
Write data to the buffer.
Definition: bitstream.cpp:237
ibitstream::next_word
void next_word()
Move to next word in buffer, refilling it if needed.
Definition: bitstream.cpp:529
bitstream::buffptr
union bitstream::@0 buffptr
Pointer to the current word.
ibitstream::windup
void windup()
Put the current word to the buffer with upper 0 bits as needed.
Definition: bitstream.cpp:543
obitstream::windup
void windup()
Put the current word to the buffer with upper 0 bits as needed.
Definition: bitstream.cpp:224
obitstream::writestring
void writestring(const char *str, bool write_trailing_null=false)
Write 8-bit character strings to the buffer.
Definition: bitstream.cpp:273
ibitstream::fill
int fill()
Fill the read buffer and reset for more data.
Definition: bitstream.cpp:661
ibitstream::~ibitstream
~ibitstream()
Destructor only uses base class destructor.
Definition: bitstream.cpp:479
obitstream::print
virtual void print() const
Print buffer contents.
Definition: bitstream.cpp:320
bitstream::bitstream
bitstream()
Dummy default constructor.
Definition: bitstream.cpp:46
bit_size
static unsigned int bit_size(int32_t i)
Find size (on [0,32]) of the smallest # that can hold the integer i.
Definition: bitstream.h:166
obitstream::write_unary
void write_unary(unsigned int value)
Write a unary code for the value.
Definition: bitstream.cpp:303
obitstream
Definition: bitstream.h:78
bitstream.h
bitstream::~bitstream
virtual ~bitstream()
Destructor deletes output buffer, closes file.
Definition: bitstream.cpp:91
bitstream::buf_used
size_t buf_used
Definition: bitstream.h:60
bitstream::get_bytes_used
virtual int get_bytes_used()
Return the number of bytes used so far in this stream.
Definition: bitstream.cpp:162
ibitstream::ibitstream
ibitstream(FILE *file, int buffersize=DEFAULT_IOBUFFER_SIZE)
Start ibitstream using a FILE ptr to an open file.
Definition: bitstream.cpp:380
ibitstream::readstring
int readstring(char *s, int count=-1)
Read a null-terminated string of 8-bit characters from the bit stream.
Definition: bitstream.cpp:590
ibitstream::print
virtual void print() const
Print buffer contents.
Definition: bitstream.cpp:636
bitstream::get_bitptr
int get_bitptr()
Get the position of the bitptr.
Definition: bitstream.cpp:170
ibitstream::close
virtual void close()
Close the IO file.
Definition: bitstream.cpp:102