NexusCPP  3.5.0
bitshuffle.h
Go to the documentation of this file.
1 /*
2  * Bitshuffle - Filter for improving compression of typed binary data.
3  *
4  * This file is part of Bitshuffle
5  * Author: Kiyoshi Masui <kiyo@physics.ubc.ca>
6  * Website: http://www.github.com/kiyo-masui/bitshuffle
7  * Created: 2014
8  *
9  * See LICENSE file for details about copyright and rights to use.
10  *
11  *
12  * Header File
13  *
14  * Worker routines return an int64_t which is the number of bytes processed
15  * if positive or an error code if negative.
16  *
17  * Error codes:
18  * -1 : Failed to allocate memory.
19  * -11 : Missing SSE.
20  * -12 : Missing AVX.
21  * -80 : Input size not a multiple of 8.
22  * -81 : block_size not multiple of 8.
23  * -91 : Decompression error, wrong number of bytes processed.
24  * -1YYY : Error internal to compression routine with error code -YYY.
25  */
26 
27 
28 #ifndef BITSHUFFLE_H
29 #define BITSHUFFLE_H
30 
31 #include <stdlib.h>
32 #include "bitshuffle_core.h"
33 
34 #ifdef __cplusplus
35 extern "C" {
36 #endif
37 
38 /* ---- bshuf_compress_lz4_bound ----
39  *
40  * Bound on size of data compressed with *bshuf_compress_lz4*.
41  *
42  * Parameters
43  * ----------
44  * size : number of elements in input
45  * elem_size : element size of typed data
46  * block_size : Process in blocks of this many elements. Pass 0 to
47  * select automatically (recommended).
48  *
49  * Returns
50  * -------
51  * Bound on compressed data size.
52  *
53  */
54 size_t bshuf_compress_lz4_bound(const size_t size,
55  const size_t elem_size, size_t block_size);
56 
57 
58 /* ---- bshuf_compress_lz4 ----
59  *
60  * Bitshuffled and compress the data using LZ4.
61  *
62  * Transpose within elements, in blocks of data of *block_size* elements then
63  * compress the blocks using LZ4. In the output buffer, each block is prefixed
64  * by a 4 byte integer giving the compressed size of that block.
65  *
66  * Output buffer must be large enough to hold the compressed data. This could
67  * be in principle substantially larger than the input buffer. Use the routine
68  * *bshuf_compress_lz4_bound* to get an upper limit.
69  *
70  * Parameters
71  * ----------
72  * in : input buffer, must be of size * elem_size bytes
73  * out : output buffer, must be large enough to hold data.
74  * size : number of elements in input
75  * elem_size : element size of typed data
76  * block_size : Process in blocks of this many elements. Pass 0 to
77  * select automatically (recommended).
78  *
79  * Returns
80  * -------
81  * number of bytes used in output buffer, negative error-code if failed.
82  *
83  */
84 int64_t bshuf_compress_lz4(const void* in, void* out, const size_t size, const size_t
85  elem_size, size_t block_size);
86 
87 
88 /* ---- bshuf_decompress_lz4 ----
89  *
90  * Undo compression and bitshuffling.
91  *
92  * Decompress data then un-bitshuffle it in blocks of *block_size* elements.
93  *
94  * To properly unshuffle bitshuffled data, *size*, *elem_size* and *block_size*
95  * must patch the parameters used to compress the data.
96  *
97  * NOT TO BE USED WITH UNTRUSTED DATA: This routine uses the function
98  * LZ4_decompress_fast from LZ4, which does not protect against maliciously
99  * formed datasets. By modifying the compressed data, this function could be
100  * coerced into leaving the boundaries of the input buffer.
101  *
102  * Parameters
103  * ----------
104  * in : input buffer
105  * out : output buffer, must be of size * elem_size bytes
106  * size : number of elements in input
107  * elem_size : element size of typed data
108  * block_size : Process in blocks of this many elements. Pass 0 to
109  * select automatically (recommended).
110  *
111  * Returns
112  * -------
113  * number of bytes consumed in *input* buffer, negative error-code if failed.
114  *
115  */
116 int64_t bshuf_decompress_lz4(const void* in, void* out, const size_t size,
117  const size_t elem_size, size_t block_size);
118 
119 #ifdef __cplusplus
120 } // extern "C"
121 #endif
122 
123 #endif // BITSHUFFLE_H
int64_t bshuf_compress_lz4(const void *in, void *out, const size_t size, const size_t elem_size, size_t block_size)
int64_t bshuf_decompress_lz4(const void *in, void *out, const size_t size, const size_t elem_size, size_t block_size)
size_t bshuf_compress_lz4_bound(const size_t size, const size_t elem_size, size_t block_size)