Update applyChunkConfiguration and dependent functions to work with n…

…ew template

+io/+config/+internal/computeChunkSizeFromConfig.m

@@ -23,8 +23,13 @@
     numDimensions = numel(dataSize);
 
     % Extract relevant configuration parameters
-    chunkDimensions = chunkSpecification.data.chunk_dimensions;
-    defaultChunkSize = chunkSpecification.chunk_default_size; % in bytes
+    chunkDimensions = chunkSpecification.chunk_dimensions;
+    if iscell(chunkDimensions)
+        numChunkDimensions = cellfun(@numel, chunkDimensions);
+        chunkDimensions = chunkDimensions{numChunkDimensions == numDimensions};
+    end
+
+    defaultChunkSize = chunkSpecification.target_chunk_size.value; % in bytes
     dataByteSize = io.config.internal.getDataByteSize(A);
 
     % Initialize chunk size array

+io/+config/+internal/configureDataPipeFromData.m

@@ -0,0 +1,41 @@
+function dataPipe = configureDataPipeFromData(numericData, datasetConfig)
+% configureDataPipeFromData - Configure a DataPipe from numeric data and dataset configuration
+
+    import io.config.internal.computeChunkSizeFromConfig
+    import types.untyped.datapipe.properties.DynamicFilter
+
+    chunkSize = computeChunkSizeFromConfig(numericData, datasetConfig);
+    maxSize = size(numericData);
+
+    dataPipeArgs = {...
+        "data", numericData, ...
+        "maxSize", maxSize, ...
+        "chunkSize", chunkSize };
+
+    hasShuffle = contains(datasetConfig.compression.prefilters, 'shuffle');
+
+    if strcmpi(datasetConfig.compression.algorithm, "Deflate")
+        % Use standard compression filters
+        dataPipeArgs = [ dataPipeArgs, ...
+            {'hasShuffle', hasShuffle, ...
+            'compressionLevel', datasetConfig.compression.level} ...
+            ];
+    else 
+        % Create property list of custom filters for dataset creation
+        compressionFilter = DynamicFilter( ...
+            datasetConfig.compression.algorithm, ...
+            datasetConfig.compression.level );
+
+        if hasShuffle
+            shuffleFilter = types.untyped.datapipe.properties.Shuffle();
+            filters = [shuffleFilter compressionFilter];
+        else
+            filters = compressionFilter;
+        end
+        dataPipeArgs = [ dataPipeArgs, ...
+            {'filters', filters} ];
+    end
+
+    % Create the datapipe.
+    dataPipe = types.untyped.DataPipe( dataPipeArgs{:} );
+end

+io/+config/+internal/reconfigureDataPipe.m

@@ -0,0 +1,4 @@
+function dataPipe = reconfigureDataPipe(dataPipe, datasetConfig)
+    % todo
+end
+

+io/+config/+internal/resolveDataTypeChunkConfig.m

@@ -1,4 +1,4 @@
-function resolvedOptions = resolveDataTypeChunkConfig(chunkSpecification, nwbObject)
+function resolvedOptions = resolveDataTypeChunkConfig(chunkSpecification, nwbObject, datasetName)
 % resolveDataTypeChunkConfig - Resolve the chunk options for individual datatypes
 %   This function resolves the chunk configuration options for a given NWB object
 %   by traversing the object hierarchy and combining options from the most specific
@@ -14,10 +14,11 @@
     arguments
         chunkSpecification (1,1) struct
         nwbObject (1,1) types.untyped.MetaClass
+        datasetName (1,1) string
     end
 
-    % Initialize resolvedOptions with an empty struct
-    resolvedOptions = struct();
+    % Initialize resolvedOptions with default options.
+    resolvedOptions = chunkSpecification.Default;
 
     % Get the NWB object type hierarchy (from most specific to base type)
     typeHierarchy = getTypeHierarchy(nwbObject);
@@ -26,12 +27,16 @@
     for i = numel(typeHierarchy):-1:1
         typeName = typeHierarchy{i};
 
-        % Check if the type has a chunkSpecification
+        % Check if the neurodata type has a chunkSpecification
         if isfield(chunkSpecification, typeName)
             typeOptions = chunkSpecification.(typeName);
 
-            % Merge options into resolvedOptions
-            resolvedOptions = mergeStructs(resolvedOptions, typeOptions);
+            % Is datasetName part of typeOptions?
+            if isfield(typeOptions, datasetName)
+                % Merge options into resolvedOptions
+                datasetOptions = typeOptions.(datasetName);
+                resolvedOptions = mergeStructs(resolvedOptions, datasetOptions);
+            end
         end
     end
 end

+io/+config/applyChunkConfiguration.m

@@ -1,49 +1,90 @@
-function applyChunkConfiguration(nwbObject, chunkConfiguration)
+function applyChunkConfiguration(nwbObject, chunkConfiguration, options)
+% applyChunkConfiguration - Apply chunk configuration to datasets of an NWB object
+
     arguments
-        nwbObject (1,1) NwbFile
-        chunkConfiguration (1,1) struct = io.config.readDefaultChunkConfiguration()
+        nwbObject (1,1) types.untyped.MetaClass
+        chunkConfiguration (1,1) struct = io.config.readDefaultChunkConfiguration() % Todo: class for this...?
+        options.OverrideExisting (1,1) logical = false
     end
+
+    import io.config.internal.resolveDataTypeChunkConfig
 
-    objectMap = nwbObject.searchFor('');
-    objectKeys = objectMap.keys();
-
-    filteredObjectMap = containers.Map();
-    for i = 1:numel(objectKeys)
-        thisObjectKey = objectKeys{i};
-        thisNwbObject = objectMap(thisObjectKey);
-        if startsWith(class(thisNwbObject), "types.") && ~startsWith(class(thisNwbObject), "types.untyped")
-            filteredObjectMap(thisObjectKey) = thisNwbObject;
-        end
+    if isa(nwbObject, 'NwbFile')
+        neurodataObjects = getNeurodataObjectsFromNwbFile(nwbObject);
+    else
+        neurodataObjects = {nwbObject};
     end
-    clear objectMap
+
+    for iNeurodataObject = 1:numel(neurodataObjects)
+        thisNeurodataObject = neurodataObjects{iNeurodataObject};
+        thisNeurodataClassName = class(thisNeurodataObject);
+
+        % Need to keep track of this. A dataset can be defined across
+        % multiple levels of the class hierarchy, the lowest class should
+        % take precedence
+        processedDatasets = string.empty;
+
+        isFinished = false;
+        while ~isFinished % Iterate over type and it's ancestor types (superclasses)
+
+            datasetNames = schemes.listDatasetsOfNeurodataType( thisNeurodataClassName );
+
+            for thisDatasetName = datasetNames % Iterate over all datasets of a type...
+
+                if ismember(thisDatasetName, processedDatasets)
+                    continue
+                end
+
+                datasetConfig = resolveDataTypeChunkConfig(...
+                    chunkConfiguration, ...
+                    thisNeurodataObject, ...
+                    thisDatasetName);
 
-    objectKeys = filteredObjectMap.keys();
-    for i = 1:numel(objectKeys)
-        thisObjectKey = objectKeys{i};
-        thisNwbObject = filteredObjectMap(thisObjectKey);
-
-        % Todo: Find dataset properties where it makes sense to do chunking
-        % I.e data, timestamps etc. Can this be determined automatically,
-        % or do we need a lookup?
-
-        dataTypeChunkOptions = io.config.internal.resolveDataTypeChunkConfig(chunkConfiguration, thisNwbObject);
-
-        if isprop(thisNwbObject, 'data')
-            if isnumeric(thisNwbObject.data)
-                % Create a datapipe object for the property value.
-                dataByteSize = io.config.internal.getDataByteSize(thisNwbObject.data);
-                if dataByteSize > dataTypeChunkOptions.chunk_default_size
-                    chunkSize = io.config.internal.computeChunkSizeFromConfig(thisNwbObject.data, dataTypeChunkOptions);
-                    maxSize = size(thisNwbObject.data);
-
-                    dataPipe = types.untyped.DataPipe( ...
-                        'data', thisNwbObject.data, ...
-                        'maxSize', maxSize, ...
-                        'chunkSize', chunkSize, ...
-                        'compressionLevel', dataTypeChunkOptions.chunk_compression_args);
-                    thisNwbObject.data = dataPipe;
+                datasetData = thisNeurodataObject.(thisDatasetName);
+
+                if isnumeric(datasetData)
+                    % Create a datapipe object for a numeric dataset value.
+                    dataByteSize = io.config.internal.getDataByteSize(datasetData);
+                    if dataByteSize > datasetConfig.target_chunk_size.value
+                        dataPipe = io.config.internal.configureDataPipeFromData(datasetData, datasetConfig);
+                    end
+                elseif isa(datasetData, 'types.untyped.DataPipe')
+                    if options.OverrideExisting
+                        dataPipe = io.config.internal.reconfigureDataPipe(datasetData, datasetConfig);
+                    end
+                elseif isa(datasetData, 'types.untyped.DataStub')
+                    % pass
+                    %error('Not implemented for files obtained by nwbRead')
+                else
+                    disp( class(datasetData) )
                 end
+
+                if exist('dataPipe', 'var')
+                    thisNeurodataObject.(thisDatasetName) = dataPipe;
+                    processedDatasets = [processedDatasets, thisDatasetName]; %#ok<AGROW>
+                    clear dataPipe
+                end
+            end
+
+            parentType = matnwb.common.getParentType(thisNeurodataClassName);
+
+            if isempty(parentType)
+                isFinished = true;
+            else
+                thisNeurodataClassName = parentType;
             end
         end
     end
 end
+
+function neurodataObjects = getNeurodataObjectsFromNwbFile(nwbObject)
+% getNeurodataObjectsFromNwbObject - Return all neurodata objects in a NwbFile object
+
+    objectMap = nwbObject.searchFor('types.');
+
+    neurodataObjects = objectMap.values();
+    neurodataClassNames = cellfun(@(c) class(c), neurodataObjects, 'uni', 0); 
+
+    toIgnore = startsWith(neurodataClassNames, "types.untyped");
+    neurodataObjects(toIgnore) = [];
+end

+io/+config/readDefaultChunkConfiguration.m

@@ -1,19 +1,24 @@
 function configObject = readDefaultChunkConfiguration()
 % READDEFAULTCHUNKCONFIGURATION Reads the default chunking configuration from a JSON file.
 %
-%   configObject = READDEFAULTCHUNKCONFIGURATION() loads the default chunking
-%   parameters from a JSON configuration file located in the 'configuration'
-%   directory within the MatNWB directory.
+% Syntax:
+%   configObject = io.config.READDEFAULTCHUNKCONFIGURATION() loads the default 
+%   chunking parameters from a JSON configuration file located in the
+%   "configuration" folder inside the MatNWB directory.
 %
-%   Output:
-%       configObject - A MATLAB structure containing the chunking parameters
+% Output Arguments:
+%   - configObject - A MATLAB structure containing the chunking parameters
 %                      defined in the JSON configuration file.
 %
-%   Example:
-%       % Load the default chunk configuration
-%       config = readDefaultChunkConfiguration();
-%       disp(config);
+% Example 1 - Load default dataset configurations::
+%    % Load the default chunk configuration
+%    config = readDefaultChunkConfiguration();
+%    disp(config);
+
+    configFilePath = fullfile(...
+        misc.getMatnwbDir, ...
+        'configuration', ...
+        'cloud_dataset_configuration.json');
 
-    configFilePath = fullfile(misc.getMatnwbDir, 'configuration', 'chunk_params.json');
     configObject = jsondecode(fileread(configFilePath));
 end

+matnwb/+common/getParentType.m

@@ -0,0 +1,7 @@
+function parentTypeClassName = getParentType(typeClassName)
+    mc = meta.class.fromName(typeClassName);
+    parentTypeClassName = mc.SuperclassList(1).Name;
+    if strcmp(parentTypeClassName, "types.untyped.MetaClass")
+        parentTypeClassName = string.empty;
+    end
+end

+schemes/listDatasetsOfNeurodataType.m

@@ -18,15 +18,33 @@
 
     assert(~isempty(typesIdx), 'Expected class name to contain "types"')
     namespaceName = classNameSplit(typesIdx+1);
+    namespaceName = strrep(namespaceName, '_', '-');
     namespace = schemes.loadNamespace(namespaceName, misc.getMatnwbDir);
 
     neurodataTypeName = classNameSplit(typesIdx+2);
     typeScheme = namespace.registry(neurodataTypeName);
 
-    datasetMaps = typeScheme('datasets');
+    switch typeScheme('class_type')
+        case 'groups'
+            if isKey(typeScheme, 'datasets')
+                datasetMaps = typeScheme('datasets');
+
+                datasetNames = repmat("", size(datasetMaps));
+                for i = 1:numel(datasetMaps)
+                    if isKey(datasetMaps{i}, 'name')
+                        datasetNames(i) = datasetMaps{i}('name');
+                    else
+                        keyboard
+                    end
+                end
+                datasetNames(datasetNames=="") = [];
+            else
+                datasetNames = string.empty;
+            end
 
-    datasetNames = repmat("", size(datasetMaps));
-    for i = 1:numel(datasetMaps)
-        datasetNames(i) = datasetMaps{i}('name');
+        case 'datasets'
+            datasetNames = "data";
+        otherwise
+            error('Unexpected class type')
     end
 end