brotli/enc/
mod.rs

1#![cfg_attr(not(feature = "std"), allow(unused_imports))]
2#[macro_use]
3pub mod vectorization;
4pub mod backward_references;
5pub mod bit_cost;
6pub mod block_split;
7pub mod brotli_bit_stream;
8pub mod cluster;
9pub mod combined_alloc;
10pub mod command;
11pub mod constants;
12pub mod dictionary_hash;
13pub mod entropy_encode;
14pub mod fast_log;
15pub mod histogram;
16pub mod input_pair;
17pub mod literal_cost;
18pub mod static_dict;
19pub mod static_dict_lut;
20pub mod utf8_util;
21pub mod util;
22pub use self::backward_references::hash_to_binary_tree;
23pub use self::backward_references::hq as backward_references_hq;
24pub mod block_splitter;
25pub mod compress_fragment;
26pub mod compress_fragment_two_pass;
27pub mod context_map_entropy;
28pub mod encode;
29pub mod find_stride;
30pub mod interface;
31pub mod ir_interpret;
32pub mod metablock;
33pub mod pdf;
34pub mod prior_eval;
35pub mod reader;
36pub mod stride_eval;
37pub mod writer;
38pub use self::combined_alloc::{BrotliAlloc, CombiningAllocator};
39mod compat;
40pub mod fixed_queue;
41pub mod multithreading;
42pub mod singlethreading;
43pub mod threading;
44pub mod worker_pool;
45#[cfg(feature = "simd")]
46pub type s16 = core::simd::i16x16;
47#[cfg(feature = "simd")]
48pub type v8 = core::simd::f32x8;
49#[cfg(feature = "simd")]
50pub type s8 = core::simd::i32x8;
51#[cfg(not(feature = "simd"))]
52pub type s16 = compat::Compat16x16;
53#[cfg(not(feature = "simd"))]
54pub type v8 = compat::CompatF8;
55#[cfg(not(feature = "simd"))]
56pub type s8 = compat::Compat32x8;
57
58mod parameters;
59mod test;
60mod weights;
61pub use self::backward_references::{BrotliEncoderParams, UnionHasher};
62use self::encode::{BrotliEncoderDestroyInstance, BrotliEncoderOperation};
63pub use self::encode::{
64    BrotliEncoderInitParams, BrotliEncoderMaxCompressedSize, BrotliEncoderMaxCompressedSizeMulti,
65};
66pub use self::hash_to_binary_tree::ZopfliNode;
67pub use self::interface::StaticCommand;
68pub use self::pdf::PDF;
69pub use self::util::floatX;
70pub use self::vectorization::{v256, v256i, Mem256f};
71use brotli_decompressor::{CustomRead, CustomWrite};
72pub use interface::{InputPair, InputReference, InputReferenceMut};
73
74pub use alloc::{AllocatedStackMemory, Allocator, SliceWrapper, SliceWrapperMut, StackAllocator};
75#[cfg(feature = "std")]
76pub use alloc_stdlib::StandardAlloc;
77#[cfg(feature = "std")]
78use std::io;
79#[cfg(feature = "std")]
80use std::io::{Error, ErrorKind, Read, Write};
81
82#[cfg(feature = "std")]
83pub use brotli_decompressor::{IntoIoReader, IoReaderWrapper, IoWriterWrapper};
84use enc::encode::BrotliEncoderStateStruct;
85
86#[cfg(not(feature = "std"))]
87pub use self::singlethreading::{compress_worker_pool, new_work_pool, WorkerPool};
88pub use self::threading::{
89    BatchSpawnableLite, BrotliEncoderThreadError, CompressionThreadResult, Owned, SendAlloc,
90};
91#[cfg(feature = "std")]
92pub use self::worker_pool::{compress_worker_pool, new_work_pool, WorkerPool};
93#[cfg(feature = "std")]
94pub fn compress_multi<
95    Alloc: BrotliAlloc + Send + 'static,
96    SliceW: SliceWrapper<u8> + Send + 'static + Sync,
97>(
98    params: &BrotliEncoderParams,
99    owned_input: &mut Owned<SliceW>,
100    output: &mut [u8],
101    alloc_per_thread: &mut [SendAlloc<
102        CompressionThreadResult<Alloc>,
103        backward_references::UnionHasher<Alloc>,
104        Alloc,
105        <WorkerPool<
106            CompressionThreadResult<Alloc>,
107            backward_references::UnionHasher<Alloc>,
108            Alloc,
109            (SliceW, BrotliEncoderParams),
110        > as BatchSpawnableLite<
111            CompressionThreadResult<Alloc>,
112            backward_references::UnionHasher<Alloc>,
113            Alloc,
114            (SliceW, BrotliEncoderParams),
115        >>::JoinHandle,
116    >],
117) -> Result<usize, BrotliEncoderThreadError>
118where
119    <Alloc as Allocator<u8>>::AllocatedMemory: Send,
120    <Alloc as Allocator<u16>>::AllocatedMemory: Send + Sync,
121    <Alloc as Allocator<u32>>::AllocatedMemory: Send + Sync,
122{
123    let mut work_pool = self::worker_pool::new_work_pool(alloc_per_thread.len() - 1);
124    compress_worker_pool(
125        params,
126        owned_input,
127        output,
128        alloc_per_thread,
129        &mut work_pool,
130    )
131}
132
133#[cfg(feature = "std")]
134pub use self::multithreading::compress_multi as compress_multi_no_threadpool;
135#[cfg(not(feature = "std"))]
136pub use self::singlethreading::compress_multi;
137#[cfg(not(feature = "std"))]
138pub use self::singlethreading::compress_multi as compress_multi_no_threadpool;
139
140#[cfg(feature = "std")]
141pub fn BrotliCompress<InputType, OutputType>(
142    r: &mut InputType,
143    w: &mut OutputType,
144    params: &BrotliEncoderParams,
145) -> Result<usize, io::Error>
146where
147    InputType: Read,
148    OutputType: Write,
149{
150    let mut input_buffer: [u8; 4096] = [0; 4096];
151    let mut output_buffer: [u8; 4096] = [0; 4096];
152    BrotliCompressCustomAlloc(
153        r,
154        w,
155        &mut input_buffer[..],
156        &mut output_buffer[..],
157        params,
158        StandardAlloc::default(),
159    )
160}
161
162#[cfg(feature = "std")]
163pub fn BrotliCompressCustomAlloc<InputType, OutputType, Alloc: BrotliAlloc>(
164    r: &mut InputType,
165    w: &mut OutputType,
166    input_buffer: &mut [u8],
167    output_buffer: &mut [u8],
168    params: &BrotliEncoderParams,
169    alloc: Alloc,
170) -> Result<usize, io::Error>
171where
172    InputType: Read,
173    OutputType: Write,
174{
175    let mut nop_callback = |_data: &mut interface::PredictionModeContextMap<InputReferenceMut>,
176                            _cmds: &mut [interface::StaticCommand],
177                            _mb: interface::InputPair,
178                            _m: &mut Alloc| ();
179    BrotliCompressCustomIo(
180        &mut IoReaderWrapper::<InputType>(r),
181        &mut IoWriterWrapper::<OutputType>(w),
182        input_buffer,
183        output_buffer,
184        params,
185        alloc,
186        &mut nop_callback,
187        Error::new(ErrorKind::UnexpectedEof, "Unexpected EOF"),
188    )
189}
190
191pub fn BrotliCompressCustomIo<
192    ErrType,
193    InputType,
194    OutputType,
195    Alloc: BrotliAlloc,
196    MetablockCallback: FnMut(
197        &mut interface::PredictionModeContextMap<InputReferenceMut>,
198        &mut [interface::StaticCommand],
199        interface::InputPair,
200        &mut Alloc,
201    ),
202>(
203    r: &mut InputType,
204    w: &mut OutputType,
205    input_buffer: &mut [u8],
206    output_buffer: &mut [u8],
207    params: &BrotliEncoderParams,
208    alloc: Alloc,
209    metablock_callback: &mut MetablockCallback,
210    unexpected_eof_error_constant: ErrType,
211) -> Result<usize, ErrType>
212where
213    InputType: CustomRead<ErrType>,
214    OutputType: CustomWrite<ErrType>,
215{
216    BrotliCompressCustomIoCustomDict(
217        r,
218        w,
219        input_buffer,
220        output_buffer,
221        params,
222        alloc,
223        metablock_callback,
224        &[],
225        unexpected_eof_error_constant,
226    )
227}
228pub fn BrotliCompressCustomIoCustomDict<
229    ErrType,
230    InputType,
231    OutputType,
232    Alloc: BrotliAlloc,
233    MetablockCallback: FnMut(
234        &mut interface::PredictionModeContextMap<InputReferenceMut>,
235        &mut [interface::StaticCommand],
236        interface::InputPair,
237        &mut Alloc,
238    ),
239>(
240    r: &mut InputType,
241    w: &mut OutputType,
242    input_buffer: &mut [u8],
243    output_buffer: &mut [u8],
244    params: &BrotliEncoderParams,
245    alloc: Alloc,
246    metablock_callback: &mut MetablockCallback,
247    dict: &[u8],
248    unexpected_eof_error_constant: ErrType,
249) -> Result<usize, ErrType>
250where
251    InputType: CustomRead<ErrType>,
252    OutputType: CustomWrite<ErrType>,
253{
254    assert!(!input_buffer.is_empty());
255    assert!(!output_buffer.is_empty());
256    let mut s_orig = BrotliEncoderStateStruct::new(alloc);
257    s_orig.params = params.clone();
258    if !dict.is_empty() {
259        s_orig.set_custom_dictionary(dict.len(), dict);
260    }
261    let mut next_in_offset: usize = 0;
262    let mut next_out_offset: usize = 0;
263    let mut total_out = Some(0);
264    let mut read_err: Result<(), ErrType> = Ok(());
265    {
266        let s = &mut s_orig;
267
268        //BrotliEncoderSetParameter(s, BrotliEncoderParameter::BROTLI_PARAM_MODE, 0 as u32); // gen, text, font
269        //BrotliEncoderSetParameter(s,
270        //                          BrotliEncoderParameter::BROTLI_PARAM_SIZE_HINT,
271        //                          input.len() as u32);
272        let mut available_in: usize = 0;
273        let mut available_out: usize = output_buffer.len();
274        let mut eof = false;
275        loop {
276            if available_in == 0 && !eof {
277                next_in_offset = 0;
278                match r.read(input_buffer) {
279                    Err(e) => {
280                        read_err = Err(e);
281                        available_in = 0;
282                        eof = true;
283                    }
284                    Ok(size) => {
285                        if size == 0 {
286                            eof = true;
287                        }
288                        available_in = size;
289                    }
290                }
291            }
292            let op: BrotliEncoderOperation;
293            if available_in == 0 {
294                op = BrotliEncoderOperation::BROTLI_OPERATION_FINISH;
295            } else {
296                op = BrotliEncoderOperation::BROTLI_OPERATION_PROCESS;
297            }
298            let result = s.compress_stream(
299                op,
300                &mut available_in,
301                input_buffer,
302                &mut next_in_offset,
303                &mut available_out,
304                output_buffer,
305                &mut next_out_offset,
306                &mut total_out,
307                metablock_callback,
308            );
309            let fin = s.is_finished();
310            if available_out == 0 || fin {
311                let lim = output_buffer.len() - available_out;
312                assert_eq!(next_out_offset, lim);
313                next_out_offset = 0;
314                while next_out_offset < lim {
315                    match w.write(&mut output_buffer[next_out_offset..lim]) {
316                        Err(e) => {
317                            BrotliEncoderDestroyInstance(s);
318                            read_err?;
319                            return Err(e);
320                        }
321                        Ok(size) => {
322                            next_out_offset += size;
323                        }
324                    }
325                }
326                available_out = output_buffer.len();
327                next_out_offset = 0;
328            }
329            if !result {
330                if read_err.is_ok() {
331                    read_err = Err(unexpected_eof_error_constant);
332                }
333                break;
334            }
335            if fin {
336                break;
337            }
338        }
339        BrotliEncoderDestroyInstance(s);
340    }
341    read_err?;
342    Ok(total_out.unwrap())
343}
brotli/enc/mod.rs

brotli/enc/
mod.rs
