Skip to content

Commit ffbfdb2

Browse files
ehennestadehennestad
authored
Change DataPipe shape validation to rely on maxSize instead of actual size. (#716)
* Create validateShape.m Added new function for validating shape of a dataset or attribute value. Extracted a static template portion from fillValidator and added a case for DataPipe values. * Update validateShape.m * Update checkDims.m Add optional argument to enforce strict check on 1D shapes, i.e not allowing shape (inf, 1) or (1, inf) etc * Update fillValidators.m Extract code "template" to another function that can be called from a validator function. The code template is static and does not need to be duplicated across many classes * Regenerated classes after updating fillValidator * Update configureDataPipeFromData.m (#717) Fixed bug where maxSize should be explicitly set for DataPipe vector data * Update validateShape.m Added extra validation of actual size/shape of data pipe objects * Update dataPipeTest.m Added unit test to test new behavior * Update dataPipeTest.m Fixed expected warning
1 parent 16862b7 commit ffbfdb2

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

47 files changed

+327
-3003
lines changed

‎+file/fillValidators.m

Lines changed: 8 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@
5757
elseif isa(prop, 'file.Attribute')
5858
unitValidationStr = strjoin({unitValidationStr...
5959
fillDtypeValidation(name, prop.dtype)...
60-
fillDimensionValidation(prop.dtype, prop.shape)...
60+
fillDimensionValidation(name, prop.dtype, prop.shape)...
6161
}, newline);
6262
else % Link
6363
fullname = namespaceReg.getFullClassName(prop.type);
@@ -154,7 +154,7 @@
154154
if isempty(prop.type)
155155
unitValidationStr = strjoin({unitValidationStr...
156156
fillDtypeValidation(name, prop.dtype)...
157-
fillDimensionValidation(prop.dtype, prop.shape)...
157+
fillDimensionValidation(name, prop.dtype, prop.shape)...
158158
}, newline);
159159
elseif prop.isConstrainedSet
160160
fullname = getFullClassName(namespaceReg, prop.type, name);
@@ -200,7 +200,7 @@
200200
);
201201
end
202202

203-
function fdvstr = fillDimensionValidation(type, shape)
203+
function fdvstr = fillDimensionValidation(name, type, shape)
204204
if strcmp(type, 'any')
205205
fdvstr = '';
206206
return;
@@ -214,33 +214,18 @@
214214
end
215215
shape{i} = ['[' strjoin(shape{i}, ',') ']'];
216216
end
217-
shapeStr = ['{' strjoin(shape, ', ') '}'];
217+
validShapeStr = ['{' strjoin(shape, ', ') '}'];
218218
else
219219
for i = 1:length(shape)
220220
shape{i} = num2str(shape{i});
221221
end
222-
shapeStr = ['{[' strjoin(shape, ',') ']}'];
222+
validShapeStr = ['{[' strjoin(shape, ',') ']}'];
223223
end
224224
else
225-
shapeStr = ['{[' num2str(shape) ']}'];
225+
validShapeStr = ['{[' num2str(shape) ']}'];
226226
end
227227

228-
fdvstr = strjoin({...
229-
'if isa(val, ''types.untyped.DataStub'')' ...
230-
' if 1 == val.ndims' ...
231-
' valsz = [val.dims 1];' ...
232-
' else' ...
233-
' valsz = val.dims;' ...
234-
' end' ...
235-
'elseif istable(val)' ...
236-
' valsz = [height(val) 1];'...
237-
'elseif ischar(val)'...
238-
' valsz = [size(val, 1) 1];'...
239-
'else'...
240-
' valsz = size(val);'...
241-
'end' ...
242-
['validshapes = ' shapeStr ';']...
243-
'types.util.checkDims(valsz, validshapes);'}, newline);
228+
fdvstr = sprintf('types.util.validateShape(''%s'', %s, val)', name, validShapeStr);
244229
end
245230

246231
%NOTE: can return empty strings
@@ -345,4 +330,4 @@
345330
['Namespace could not be found for type `%s`.' ...
346331
' Skipping Validation for property `%s`.'], propType, name);
347332
end
348-
end
333+
end

‎+io/+config/+internal/configureDataPipeFromData.m

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,13 @@
55
import types.untyped.datapipe.properties.DynamicFilter
66

77
chunkSize = computeChunkSizeFromConfig(numericData, datasetConfig.chunking);
8-
maxSize = size(numericData);
8+
if isvector(numericData)
9+
% If input data is vector, we use maxSize = Inf to enforce a 1D
10+
% columnar representation of data in file.
11+
maxSize = Inf;
12+
else
13+
maxSize = size(numericData);
14+
end
915

1016
dataPipeArgs = {...
1117
"data", numericData, ...

‎+tests/+unit/dataPipeTest.m

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -290,6 +290,29 @@ function testOverrideBoundPipeProperties(testCase)
290290
testCase.verifyEqual(ME.identifier, 'NWB:BoundPipe:CannotSetPipeProperty')
291291
end
292292
end
293+
294+
function testShapeValidation(testCase)
295+
% Create a DataPipe with both maxSize and actual size that are
296+
% valid
297+
dataPipe = types.untyped.DataPipe( 'data', rand(50, 50, 3), 'maxSize', [50,50,inf] );
298+
try
299+
imageSeries = types.core.ImageSeries('data', dataPipe, 'data_unit', 'test'); %#ok<NASGU>
300+
catch
301+
testCase.verifyFail('Expected DataPipe with valid shape for ImageSeries to pass')
302+
end
303+
% Create a DataPipe where maxSize is invalid
304+
dataPipe = types.untyped.DataPipe( 'data', rand(50, 50, 3, 4, 10), 'maxSize', [50, 50, 3, 4, inf] );
305+
testCase.verifyError(...
306+
@() types.core.ImageSeries('data', dataPipe, 'data_unit', 'test'), ...
307+
'NWB:CheckDims:InvalidDimensions')
308+
309+
% Create a DataPipe where maxSize is valid and actual size is
310+
% invalid
311+
dataPipe = types.untyped.DataPipe( 'data', rand(50, 50, 3, 4, 10), 'maxSize', [50, 50, 3, inf] );
312+
testCase.verifyWarning(...
313+
@() types.core.ImageSeries('data', dataPipe, 'data_unit', 'test'), ...
314+
'NWB:ValidateShape:InvalidDataPipeSize')
315+
end
293316
end
294317

295318
methods (Test, TestTags={'UsesDynamicallyLoadedFilters'})

‎+types/+core/AbstractFeatureSeries.m

Lines changed: 4 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -90,75 +90,19 @@
9090

9191
function val = validate_data(obj, val)
9292
val = types.util.checkDtype('data', 'numeric', val);
93-
if isa(val, 'types.untyped.DataStub')
94-
if 1 == val.ndims
95-
valsz = [val.dims 1];
96-
else
97-
valsz = val.dims;
98-
end
99-
elseif istable(val)
100-
valsz = [height(val) 1];
101-
elseif ischar(val)
102-
valsz = [size(val, 1) 1];
103-
else
104-
valsz = size(val);
105-
end
106-
validshapes = {[Inf,Inf], [Inf]};
107-
types.util.checkDims(valsz, validshapes);
93+
types.util.validateShape('data', {[Inf,Inf], [Inf]}, val)
10894
end
10995
function val = validate_data_unit(obj, val)
11096
val = types.util.checkDtype('data_unit', 'char', val);
111-
if isa(val, 'types.untyped.DataStub')
112-
if 1 == val.ndims
113-
valsz = [val.dims 1];
114-
else
115-
valsz = val.dims;
116-
end
117-
elseif istable(val)
118-
valsz = [height(val) 1];
119-
elseif ischar(val)
120-
valsz = [size(val, 1) 1];
121-
else
122-
valsz = size(val);
123-
end
124-
validshapes = {[1]};
125-
types.util.checkDims(valsz, validshapes);
97+
types.util.validateShape('data_unit', {[1]}, val)
12698
end
12799
function val = validate_feature_units(obj, val)
128100
val = types.util.checkDtype('feature_units', 'char', val);
129-
if isa(val, 'types.untyped.DataStub')
130-
if 1 == val.ndims
131-
valsz = [val.dims 1];
132-
else
133-
valsz = val.dims;
134-
end
135-
elseif istable(val)
136-
valsz = [height(val) 1];
137-
elseif ischar(val)
138-
valsz = [size(val, 1) 1];
139-
else
140-
valsz = size(val);
141-
end
142-
validshapes = {[Inf]};
143-
types.util.checkDims(valsz, validshapes);
101+
types.util.validateShape('feature_units', {[Inf]}, val)
144102
end
145103
function val = validate_features(obj, val)
146104
val = types.util.checkDtype('features', 'char', val);
147-
if isa(val, 'types.untyped.DataStub')
148-
if 1 == val.ndims
149-
valsz = [val.dims 1];
150-
else
151-
valsz = val.dims;
152-
end
153-
elseif istable(val)
154-
valsz = [height(val) 1];
155-
elseif ischar(val)
156-
valsz = [size(val, 1) 1];
157-
else
158-
valsz = size(val);
159-
end
160-
validshapes = {[Inf]};
161-
types.util.checkDims(valsz, validshapes);
105+
types.util.validateShape('features', {[Inf]}, val)
162106
end
163107
%% EXPORT
164108
function refs = export(obj, fid, fullpath, refs)

‎+types/+core/AnnotationSeries.m

Lines changed: 1 addition & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -67,21 +67,7 @@
6767

6868
function val = validate_data(obj, val)
6969
val = types.util.checkDtype('data', 'char', val);
70-
if isa(val, 'types.untyped.DataStub')
71-
if 1 == val.ndims
72-
valsz = [val.dims 1];
73-
else
74-
valsz = val.dims;
75-
end
76-
elseif istable(val)
77-
valsz = [height(val) 1];
78-
elseif ischar(val)
79-
valsz = [size(val, 1) 1];
80-
else
81-
valsz = size(val);
82-
end
83-
validshapes = {[Inf]};
84-
types.util.checkDims(valsz, validshapes);
70+
types.util.validateShape('data', {[Inf]}, val)
8571
end
8672
function val = validate_data_resolution(obj, val)
8773
if isequal(val, -1)

‎+types/+core/ClusterWaveforms.m

Lines changed: 3 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -84,57 +84,15 @@
8484
end
8585
function val = validate_waveform_filtering(obj, val)
8686
val = types.util.checkDtype('waveform_filtering', 'char', val);
87-
if isa(val, 'types.untyped.DataStub')
88-
if 1 == val.ndims
89-
valsz = [val.dims 1];
90-
else
91-
valsz = val.dims;
92-
end
93-
elseif istable(val)
94-
valsz = [height(val) 1];
95-
elseif ischar(val)
96-
valsz = [size(val, 1) 1];
97-
else
98-
valsz = size(val);
99-
end
100-
validshapes = {[1]};
101-
types.util.checkDims(valsz, validshapes);
87+
types.util.validateShape('waveform_filtering', {[1]}, val)
10288
end
10389
function val = validate_waveform_mean(obj, val)
10490
val = types.util.checkDtype('waveform_mean', 'single', val);
105-
if isa(val, 'types.untyped.DataStub')
106-
if 1 == val.ndims
107-
valsz = [val.dims 1];
108-
else
109-
valsz = val.dims;
110-
end
111-
elseif istable(val)
112-
valsz = [height(val) 1];
113-
elseif ischar(val)
114-
valsz = [size(val, 1) 1];
115-
else
116-
valsz = size(val);
117-
end
118-
validshapes = {[Inf,Inf]};
119-
types.util.checkDims(valsz, validshapes);
91+
types.util.validateShape('waveform_mean', {[Inf,Inf]}, val)
12092
end
12193
function val = validate_waveform_sd(obj, val)
12294
val = types.util.checkDtype('waveform_sd', 'single', val);
123-
if isa(val, 'types.untyped.DataStub')
124-
if 1 == val.ndims
125-
valsz = [val.dims 1];
126-
else
127-
valsz = val.dims;
128-
end
129-
elseif istable(val)
130-
valsz = [height(val) 1];
131-
elseif ischar(val)
132-
valsz = [size(val, 1) 1];
133-
else
134-
valsz = size(val);
135-
end
136-
validshapes = {[Inf,Inf]};
137-
types.util.checkDims(valsz, validshapes);
95+
types.util.validateShape('waveform_sd', {[Inf,Inf]}, val)
13896
end
13997
%% EXPORT
14098
function refs = export(obj, fid, fullpath, refs)

‎+types/+core/Clustering.m

Lines changed: 4 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -72,75 +72,19 @@
7272

7373
function val = validate_description(obj, val)
7474
val = types.util.checkDtype('description', 'char', val);
75-
if isa(val, 'types.untyped.DataStub')
76-
if 1 == val.ndims
77-
valsz = [val.dims 1];
78-
else
79-
valsz = val.dims;
80-
end
81-
elseif istable(val)
82-
valsz = [height(val) 1];
83-
elseif ischar(val)
84-
valsz = [size(val, 1) 1];
85-
else
86-
valsz = size(val);
87-
end
88-
validshapes = {[1]};
89-
types.util.checkDims(valsz, validshapes);
75+
types.util.validateShape('description', {[1]}, val)
9076
end
9177
function val = validate_num(obj, val)
9278
val = types.util.checkDtype('num', 'int32', val);
93-
if isa(val, 'types.untyped.DataStub')
94-
if 1 == val.ndims
95-
valsz = [val.dims 1];
96-
else
97-
valsz = val.dims;
98-
end
99-
elseif istable(val)
100-
valsz = [height(val) 1];
101-
elseif ischar(val)
102-
valsz = [size(val, 1) 1];
103-
else
104-
valsz = size(val);
105-
end
106-
validshapes = {[Inf]};
107-
types.util.checkDims(valsz, validshapes);
79+
types.util.validateShape('num', {[Inf]}, val)
10880
end
10981
function val = validate_peak_over_rms(obj, val)
11082
val = types.util.checkDtype('peak_over_rms', 'single', val);
111-
if isa(val, 'types.untyped.DataStub')
112-
if 1 == val.ndims
113-
valsz = [val.dims 1];
114-
else
115-
valsz = val.dims;
116-
end
117-
elseif istable(val)
118-
valsz = [height(val) 1];
119-
elseif ischar(val)
120-
valsz = [size(val, 1) 1];
121-
else
122-
valsz = size(val);
123-
end
124-
validshapes = {[Inf]};
125-
types.util.checkDims(valsz, validshapes);
83+
types.util.validateShape('peak_over_rms', {[Inf]}, val)
12684
end
12785
function val = validate_times(obj, val)
12886
val = types.util.checkDtype('times', 'double', val);
129-
if isa(val, 'types.untyped.DataStub')
130-
if 1 == val.ndims
131-
valsz = [val.dims 1];
132-
else
133-
valsz = val.dims;
134-
end
135-
elseif istable(val)
136-
valsz = [height(val) 1];
137-
elseif ischar(val)
138-
valsz = [size(val, 1) 1];
139-
else
140-
valsz = size(val);
141-
end
142-
validshapes = {[Inf]};
143-
types.util.checkDims(valsz, validshapes);
87+
types.util.validateShape('times', {[Inf]}, val)
14488
end
14589
%% EXPORT
14690
function refs = export(obj, fid, fullpath, refs)

0 commit comments

Comments
 (0)