brotli/enc/
mod.rs

1#[macro_use]
2pub mod vectorization;
3pub mod backward_references;
4pub mod bit_cost;
5pub mod block_split;
6pub mod block_splitter;
7pub mod brotli_bit_stream;
8pub mod cluster;
9pub mod combined_alloc;
10pub mod command;
11mod compat;
12pub mod compress_fragment;
13pub mod compress_fragment_two_pass;
14pub mod constants;
15pub mod context_map_entropy;
16pub mod dictionary_hash;
17pub mod encode;
18pub mod entropy_encode;
19pub mod find_stride;
20pub mod fixed_queue;
21pub mod histogram;
22pub mod input_pair;
23pub mod interface;
24pub mod ir_interpret;
25pub mod literal_cost;
26mod log_table_16;
27mod log_table_8;
28pub mod metablock;
29pub mod multithreading;
30mod parameters;
31pub mod pdf;
32pub mod prior_eval;
33pub mod reader;
34pub mod singlethreading;
35pub mod static_dict;
36pub mod static_dict_lut;
37pub mod stride_eval;
38mod test;
39pub mod threading;
40pub mod utf8_util;
41pub mod util;
42mod weights;
43pub mod worker_pool;
44pub mod writer;
45
46pub use alloc::{AllocatedStackMemory, Allocator, SliceWrapper, SliceWrapperMut, StackAllocator};
47#[cfg(feature = "std")]
48use std::io;
49#[cfg(feature = "std")]
50use std::io::{Error, ErrorKind, Read, Write};
51
52#[cfg(feature = "std")]
53pub use alloc_stdlib::StandardAlloc;
54use brotli_decompressor::{CustomRead, CustomWrite};
55#[cfg(feature = "std")]
56pub use brotli_decompressor::{IntoIoReader, IoReaderWrapper, IoWriterWrapper};
57pub use interface::{InputPair, InputReference, InputReferenceMut};
58
59pub use self::backward_references::{
60    hash_to_binary_tree, hq as backward_references_hq, BrotliEncoderParams, UnionHasher,
61};
62pub use self::combined_alloc::{BrotliAlloc, CombiningAllocator};
63use self::encode::{BrotliEncoderDestroyInstance, BrotliEncoderOperation};
64pub use self::encode::{
65    BrotliEncoderInitParams, BrotliEncoderMaxCompressedSize, BrotliEncoderMaxCompressedSizeMulti,
66};
67pub use self::hash_to_binary_tree::ZopfliNode;
68pub use self::interface::StaticCommand;
69pub use self::pdf::PDF;
70#[cfg(not(feature = "std"))]
71pub use self::singlethreading::{compress_worker_pool, new_work_pool, WorkerPool};
72pub use self::threading::{
73    BatchSpawnableLite, BrotliEncoderThreadError, CompressionThreadResult, Owned, SendAlloc,
74};
75pub use self::util::floatX;
76pub use self::vectorization::{v256, v256i, Mem256f};
77#[cfg(feature = "std")]
78pub use self::worker_pool::{compress_worker_pool, new_work_pool, WorkerPool};
79use crate::enc::encode::BrotliEncoderStateStruct;
80
81#[cfg(feature = "simd")]
82pub type s16 = core::simd::i16x16;
83#[cfg(feature = "simd")]
84pub type v8 = core::simd::f32x8;
85#[cfg(feature = "simd")]
86pub type s8 = core::simd::i32x8;
87#[cfg(not(feature = "simd"))]
88pub type s16 = compat::Compat16x16;
89#[cfg(not(feature = "simd"))]
90pub type v8 = compat::CompatF8;
91#[cfg(not(feature = "simd"))]
92pub type s8 = compat::Compat32x8;
93
94#[cfg(feature = "std")]
95pub fn compress_multi<
96    Alloc: BrotliAlloc + Send + 'static,
97    SliceW: SliceWrapper<u8> + Send + 'static + Sync,
98>(
99    params: &BrotliEncoderParams,
100    owned_input: &mut Owned<SliceW>,
101    output: &mut [u8],
102    alloc_per_thread: &mut [SendAlloc<
103        CompressionThreadResult<Alloc>,
104        backward_references::UnionHasher<Alloc>,
105        Alloc,
106        <WorkerPool<
107            CompressionThreadResult<Alloc>,
108            backward_references::UnionHasher<Alloc>,
109            Alloc,
110            (SliceW, BrotliEncoderParams),
111        > as BatchSpawnableLite<
112            CompressionThreadResult<Alloc>,
113            backward_references::UnionHasher<Alloc>,
114            Alloc,
115            (SliceW, BrotliEncoderParams),
116        >>::JoinHandle,
117    >],
118) -> Result<usize, BrotliEncoderThreadError>
119where
120    <Alloc as Allocator<u8>>::AllocatedMemory: Send,
121    <Alloc as Allocator<u16>>::AllocatedMemory: Send + Sync,
122    <Alloc as Allocator<u32>>::AllocatedMemory: Send + Sync,
123{
124    let mut work_pool = self::worker_pool::new_work_pool(alloc_per_thread.len() - 1);
125    compress_worker_pool(
126        params,
127        owned_input,
128        output,
129        alloc_per_thread,
130        &mut work_pool,
131    )
132}
133
134#[cfg(feature = "std")]
135pub use self::multithreading::compress_multi as compress_multi_no_threadpool;
136#[cfg(not(feature = "std"))]
137pub use self::singlethreading::compress_multi;
138#[cfg(not(feature = "std"))]
139pub use self::singlethreading::compress_multi as compress_multi_no_threadpool;
140
141#[cfg(feature = "std")]
142pub fn BrotliCompress<InputType, OutputType>(
143    r: &mut InputType,
144    w: &mut OutputType,
145    params: &BrotliEncoderParams,
146) -> Result<usize, io::Error>
147where
148    InputType: Read,
149    OutputType: Write,
150{
151    let mut input_buffer: [u8; 4096] = [0; 4096];
152    let mut output_buffer: [u8; 4096] = [0; 4096];
153    BrotliCompressCustomAlloc(
154        r,
155        w,
156        &mut input_buffer[..],
157        &mut output_buffer[..],
158        params,
159        StandardAlloc::default(),
160    )
161}
162
163#[cfg(feature = "std")]
164pub fn BrotliCompressCustomAlloc<InputType, OutputType, Alloc: BrotliAlloc>(
165    r: &mut InputType,
166    w: &mut OutputType,
167    input_buffer: &mut [u8],
168    output_buffer: &mut [u8],
169    params: &BrotliEncoderParams,
170    alloc: Alloc,
171) -> Result<usize, io::Error>
172where
173    InputType: Read,
174    OutputType: Write,
175{
176    let mut nop_callback = |_data: &mut interface::PredictionModeContextMap<InputReferenceMut>,
177                            _cmds: &mut [interface::StaticCommand],
178                            _mb: interface::InputPair,
179                            _m: &mut Alloc| ();
180    BrotliCompressCustomIo(
181        &mut IoReaderWrapper::<InputType>(r),
182        &mut IoWriterWrapper::<OutputType>(w),
183        input_buffer,
184        output_buffer,
185        params,
186        alloc,
187        &mut nop_callback,
188        Error::new(ErrorKind::UnexpectedEof, "Unexpected EOF"),
189    )
190}
191
192pub fn BrotliCompressCustomIo<
193    ErrType,
194    InputType,
195    OutputType,
196    Alloc: BrotliAlloc,
197    MetablockCallback: FnMut(
198        &mut interface::PredictionModeContextMap<InputReferenceMut>,
199        &mut [interface::StaticCommand],
200        interface::InputPair,
201        &mut Alloc,
202    ),
203>(
204    r: &mut InputType,
205    w: &mut OutputType,
206    input_buffer: &mut [u8],
207    output_buffer: &mut [u8],
208    params: &BrotliEncoderParams,
209    alloc: Alloc,
210    metablock_callback: &mut MetablockCallback,
211    unexpected_eof_error_constant: ErrType,
212) -> Result<usize, ErrType>
213where
214    InputType: CustomRead<ErrType>,
215    OutputType: CustomWrite<ErrType>,
216{
217    BrotliCompressCustomIoCustomDict(
218        r,
219        w,
220        input_buffer,
221        output_buffer,
222        params,
223        alloc,
224        metablock_callback,
225        &[],
226        unexpected_eof_error_constant,
227    )
228}
229pub fn BrotliCompressCustomIoCustomDict<
230    ErrType,
231    InputType,
232    OutputType,
233    Alloc: BrotliAlloc,
234    MetablockCallback: FnMut(
235        &mut interface::PredictionModeContextMap<InputReferenceMut>,
236        &mut [interface::StaticCommand],
237        interface::InputPair,
238        &mut Alloc,
239    ),
240>(
241    r: &mut InputType,
242    w: &mut OutputType,
243    input_buffer: &mut [u8],
244    output_buffer: &mut [u8],
245    params: &BrotliEncoderParams,
246    alloc: Alloc,
247    metablock_callback: &mut MetablockCallback,
248    dict: &[u8],
249    unexpected_eof_error_constant: ErrType,
250) -> Result<usize, ErrType>
251where
252    InputType: CustomRead<ErrType>,
253    OutputType: CustomWrite<ErrType>,
254{
255    assert!(!input_buffer.is_empty());
256    assert!(!output_buffer.is_empty());
257    let mut s_orig = BrotliEncoderStateStruct::new(alloc);
258    s_orig.params = params.clone();
259    if !dict.is_empty() {
260        s_orig.set_custom_dictionary(dict.len(), dict);
261    }
262    let mut next_in_offset: usize = 0;
263    let mut next_out_offset: usize = 0;
264    let mut total_out = Some(0);
265    let mut read_err: Result<(), ErrType> = Ok(());
266    {
267        let s = &mut s_orig;
268
269        //BrotliEncoderSetParameter(s, BrotliEncoderParameter::BROTLI_PARAM_MODE, 0 as u32); // gen, text, font
270        //BrotliEncoderSetParameter(s,
271        //                          BrotliEncoderParameter::BROTLI_PARAM_SIZE_HINT,
272        //                          input.len() as u32);
273        let mut available_in: usize = 0;
274        let mut available_out: usize = output_buffer.len();
275        let mut eof = false;
276        loop {
277            if available_in == 0 && !eof {
278                next_in_offset = 0;
279                match r.read(input_buffer) {
280                    Err(e) => {
281                        read_err = Err(e);
282                        available_in = 0;
283                        eof = true;
284                    }
285                    Ok(size) => {
286                        if size == 0 {
287                            eof = true;
288                        }
289                        available_in = size;
290                    }
291                }
292            }
293            let op: BrotliEncoderOperation;
294            if available_in == 0 {
295                op = BrotliEncoderOperation::BROTLI_OPERATION_FINISH;
296            } else {
297                op = BrotliEncoderOperation::BROTLI_OPERATION_PROCESS;
298            }
299            let result = s.compress_stream(
300                op,
301                &mut available_in,
302                input_buffer,
303                &mut next_in_offset,
304                &mut available_out,
305                output_buffer,
306                &mut next_out_offset,
307                &mut total_out,
308                metablock_callback,
309            );
310            let fin = s.is_finished();
311            if available_out == 0 || fin {
312                let lim = output_buffer.len() - available_out;
313                assert_eq!(next_out_offset, lim);
314                next_out_offset = 0;
315                while next_out_offset < lim {
316                    match w.write(&mut output_buffer[next_out_offset..lim]) {
317                        Err(e) => {
318                            BrotliEncoderDestroyInstance(s);
319                            read_err?;
320                            return Err(e);
321                        }
322                        Ok(size) => {
323                            next_out_offset += size;
324                        }
325                    }
326                }
327                available_out = output_buffer.len();
328                next_out_offset = 0;
329            }
330            if !result {
331                if read_err.is_ok() {
332                    read_err = Err(unexpected_eof_error_constant);
333                }
334                break;
335            }
336            if fin {
337                break;
338            }
339        }
340        BrotliEncoderDestroyInstance(s);
341    }
342    read_err?;
343    Ok(total_out.unwrap())
344}
brotli/enc/mod.rs

brotli/enc/
mod.rs
