Skip to content

Commit cf57326

Browse files
fangqfangq
committed
support blosc2 meta compressors
1 parent d37a386 commit cf57326

File tree

7 files changed

+634
-528
lines changed

7 files changed

+634
-528
lines changed

‎blosc2decode.m

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
function varargout = blosc2decode(varargin)
2+
%
3+
% output = blosc2decode(input,codec)
4+
% output = blosc2decode(input)
5+
% or
6+
% output = blosc2decode(input,info)
7+
%
8+
% Decompressing an blosc2-compressed byte-stream to recover the original data
9+
% This function depends on the ZMat toolbox (http://github.com/fangq/zmat)
10+
%
11+
% authors:Qianqian Fang (q.fang <at> neu.edu)
12+
%
13+
% input:
14+
% input: a string, int8/uint8 vector or numerical array to store blosc2-compressed data
15+
% codec: if the 2nd input is a string, it is treated as a compression method that
16+
% blosc2 supports, it can be one of:
17+
% 'blosc2blosclz', 'blosc2lz4', 'blosc2lz4hc', 'blosc2zlib' and 'blosc2zstd'
18+
% if no codec is specified, 'blosc2blosclz' method is assumed
19+
% info (optional): a struct produced by the zmat/blosc2encode function during
20+
% compression; if not given, the inputs/outputs will be treated as a
21+
% 1-D vector
22+
%
23+
% output:
24+
% output: the decompressed byte stream stored in a uint8 vector; if info is
25+
% given, output will restore the original data's type and dimensions
26+
%
27+
% examples:
28+
% [bytes, info]=blosc2encode(eye(10));
29+
% orig=blosc2decode(bytes,info);
30+
%
31+
% license:
32+
% BSD or GPL version 3, see LICENSE_{BSD,GPLv3}.txt files for details
33+
%
34+
% -- this function is part of JSONLab toolbox (http://iso2mesh.sf.net/cgi-bin/index.cgi?jsonlab)
35+
%
36+
37+
if (nargin == 0)
38+
error('you must provide at least 1 input');
39+
end
40+
if (exist('zmat', 'file') == 2 || exist('zmat', 'file') == 3)
41+
if (nargin >= 2 && ischar(varargin{2}))
42+
[varargout{1:nargout}] = zmat(varargin{1}, 0, varargin{2:end});
43+
elseif (nargin > 1)
44+
[varargout{1:nargout}] = zmat(varargin{1}, varargin{2:end});
45+
else
46+
[varargout{1:nargout}] = zmat(varargin{1}, 0, 'blosc2blosclz', varargin{2:end});
47+
end
48+
else
49+
error('you must install ZMat toolbox to use this feature: http://github.com/NeuroJSON/zmat');
50+
end

‎blosc2encode.m

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
function varargout = blosc2encode(varargin)
2+
%
3+
% output = blosc2encode(input)
4+
% or
5+
% [output, info] = blosc2encode(input)
6+
%
7+
% Compress a string or a numerical array using LZ4-compression
8+
%
9+
% This function depends on the ZMat toolbox (http://github.com/fangq/zmat)
10+
%
11+
% authors:Qianqian Fang (q.fang <at> neu.edu)
12+
%
13+
% input:
14+
% input: the original data, can be a string, a numerical vector or array
15+
%
16+
% output:
17+
% output: the compressed byte stream stored in a uint8 vector
18+
% info: (optional) a struct storing the metadata of the input, see "help zmat" for details
19+
%
20+
% examples:
21+
% [bytes, info]=blosc2encode(eye(10));
22+
% orig=blosc2decode(bytes,info);
23+
%
24+
% license:
25+
% BSD or GPL version 3, see LICENSE_{BSD,GPLv3}.txt files for details
26+
%
27+
% -- this function is part of JSONLab toolbox (http://iso2mesh.sf.net/cgi-bin/index.cgi?jsonlab)
28+
%
29+
30+
if (nargin == 0)
31+
error('you must provide at least 1 input');
32+
end
33+
34+
if (exist('zmat', 'file') == 2 || exist('zmat', 'file') == 3)
35+
if (nargin > 1 && ischar(varargin{2}))
36+
[varargout{1:nargout}] = zmat(varargin{1}, 1, varargin{2:end});
37+
else
38+
[varargout{1:nargout}] = zmat(varargin{1}, 1, 'blosc2blosclz', varargin{2:end});
39+
end
40+
return
41+
else
42+
error('you must install ZMat toolbox to use this feature: http://github.com/NeuroJSON/zmat');
43+
end

‎jdatadecode.m

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -126,18 +126,24 @@
126126
if(isfield(data,N_('_ArrayZipType_')))
127127
zipmethod=data(j).(N_('_ArrayZipType_'));
128128
end
129-
if(ismember(zipmethod,{'zlib','gzip','lzma','lzip','lz4','lz4hc','base64'}))
130-
decompfun=str2func([zipmethod 'decode']);
129+
if(ismember(zipmethod,{'zlib','gzip','lzma','lzip','lz4','lz4hc','base64'}) || ~isempty(regexp(zipmethod,'^blosc2', 'once')))
130+
decodeparam={};
131+
if(~isempty(regexp(zipmethod,'^blosc2', 'once')))
132+
decompfun=@blosc2decode;
133+
decodeparam={zipmethod};
134+
else
135+
decompfun=str2func([zipmethod 'decode']);
136+
end
131137
arraytype=data(j).(N_('_ArrayType_'));
132138
chartype=0;
133139
if(strcmp(arraytype,'char') || strcmp(arraytype,'logical'))
134140
chartype=1;
135141
arraytype='uint8';
136142
end
137143
if(needbase64 && strcmp(zipmethod,'base64')==0)
138-
ndata=reshape(typecast(decompfun(base64decode(data(j).(N_('_ArrayZipData_')))),arraytype),dims);
144+
ndata=reshape(typecast(decompfun(base64decode(data(j).(N_('_ArrayZipData_'))),decodeparam{:}),arraytype),dims);
139145
else
140-
ndata=reshape(typecast(decompfun(data(j).(N_('_ArrayZipData_'))),arraytype),dims);
146+
ndata=reshape(typecast(decompfun(data(j).(N_('_ArrayZipData_')),decodeparam{:}),arraytype),dims);
141147
end
142148
if(chartype)
143149
ndata=char(ndata);

‎jdataencode.m

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -314,12 +314,16 @@
314314
end
315315

316316
if(~isempty(zipmethod) && numel(item)>minsize)
317-
compfun=str2func([zipmethod 'encode']);
317+
compfun=str2func([regexprep(zipmethod,'blosc2[a-z0-9]+','blosc2') 'encode']);
318318
newitem.(N('_ArrayZipType_'))=lower(zipmethod);
319319
if(~isfield(newitem,N('_ArrayZipSize_')))
320320
newitem.(N('_ArrayZipSize_'))=size(newitem.(N('_ArrayData_')));
321321
end
322-
newitem.(N('_ArrayZipData_'))=compfun(typecast(newitem.(N('_ArrayData_'))(:).','uint8'));
322+
if(strcmp(zipmethod,'blosc2'))
323+
newitem.(N('_ArrayZipData_'))=compfun(typecast(newitem.(N('_ArrayData_'))(:).','uint8'), zipmethod);
324+
else
325+
newitem.(N('_ArrayZipData_'))=compfun(typecast(newitem.(N('_ArrayData_'))(:).','uint8'));
326+
end
323327
newitem=rmfield(newitem,N('_ArrayData_'));
324328
if(varargin{1}.base64)
325329
newitem.(N('_ArrayZipData_'))=char(base64encode(newitem.(N('_ArrayZipData_'))));

‎savebj.m

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -192,7 +192,7 @@
192192

193193
dozip=opt.compression;
194194
if(~isempty(dozip))
195-
if(~ismember(dozip,{'zlib','gzip','lzma','lzip','lz4','lz4hc'}))
195+
if (~ismember(dozip, {'zlib', 'gzip', 'lzma', 'lzip', 'lz4', 'lz4hc'}) && isempty(regexp(dozip,'^blosc2', 'once')))
196196
error('compression method "%s" is not supported',dozip);
197197
end
198198
if(exist('zmat','file')~=2 && exist('zmat','file')~=3)

0 commit comments

Comments
 (0)