Add functionality for installing extensions (#637)

* Add functionality for installing extensions

* Minor fixes

* Update comment

* Add comment + print message when extension has been installed

* Update installExtension.m

* Update matnwb_createNwbInstallExtension.m

Update docstring

* Add workflow for updating nwbInstallExtension

* Add option to save extension in custom location

* Create InstallExtensionTest.m

* Update docstring

* Change dispExtensionInfo to return info instead of displaying + add test

* Reorganize code into separate functions and add tests

* Minor changes to improve test coverage

* add nwbInstallExtension to docs

* Update update_extension_list.yml

Add schedule event for workflow to update nwbInstallExtension

* Update downloadExtensionRepository.m

Remove local function

* Update docstring for nwbInstallExtension

* Fix docstring indentation in nwbInstallExtension

* Add doc pages describing how to use (ndx) extensions

* Fix typo

* Update +tests/+unit/InstallExtensionTest.m

Co-authored-by: Ben Dichter <[email protected]>

* Update docs/source/pages/getting_started/using_extensions/generating_extension_api.rst

Co-authored-by: Ben Dichter <[email protected]>

* Add docstrings for functions to retrieve and list extension info

* Fix docstring formatting/whitespace

* Update listExtensions.m

Add example to docstring

* Update installing_extensions.rst

---------

Co-authored-by: Ben Dichter <[email protected]>

+matnwb/+extension/+internal/buildRepoDownloadUrl.m

@@ -0,0 +1,25 @@
+function downloadUrl = buildRepoDownloadUrl(repositoryUrl, branchName)
+% buildRepoDownloadUrl - Build a download URL for a given repository and branch
+    arguments
+        repositoryUrl (1,1) string
+        branchName (1,1) string
+    end
+
+    if endsWith(repositoryUrl, '/')
+        repositoryUrl = extractBefore(repositoryUrl, strlength(repositoryUrl));
+    end
+
+    if contains(repositoryUrl, 'github.com')
+        downloadUrl = sprintf( '%s/archive/refs/heads/%s.zip', repositoryUrl, branchName );
+
+    elseif contains(repositoryUrl, 'gitlab.com')
+        repoPathSegments = strsplit(repositoryUrl, '/');
+        repoName = repoPathSegments{end};
+        downloadUrl = sprintf( '%s/-/archive/%s/%s-%s.zip', ...
+            repositoryUrl, branchName, repoName, branchName);
+
+    else
+        error('NWB:BuildRepoDownloadUrl:UnsupportedRepository', ...
+            'Expected repository URL to point to a GitHub or a GitLab repository')
+    end
+end

+matnwb/+extension/+internal/downloadExtensionRepository.m

@@ -0,0 +1,46 @@
+function [wasDownloaded, repoTargetFolder] = downloadExtensionRepository(...
+    repositoryUrl, repoTargetFolder, extensionName)
+% downloadExtensionRepository - Download the repository (source) for an extension
+%
+%   The metadata for a neurodata extension only provides the url to the
+%   repository containing the extension, not the full download url. This
+%   function tries to download a zipped version of the repository from
+%   either the "main" or the "master" branch.
+%
+%   Works for repositories located on GitHub or GitLab
+%
+%   As of Dec. 2024, this approach works for all registered extensions
+
+    arguments
+        repositoryUrl (1,1) string
+        repoTargetFolder (1,1) string
+        extensionName (1,1) string
+    end
+
+    import matnwb.extension.internal.downloadZippedRepo
+    import matnwb.extension.internal.buildRepoDownloadUrl
+
+    defaultBranchNames = ["main", "master"];
+
+    wasDownloaded = false;
+    for i = 1:2
+        try
+            branchName = defaultBranchNames(i);
+            downloadUrl = buildRepoDownloadUrl(repositoryUrl, branchName);
+            repoTargetFolder = downloadZippedRepo(downloadUrl, repoTargetFolder);
+            wasDownloaded = true;
+            break
+        catch ME
+            if strcmp(ME.identifier, 'MATLAB:webservices:HTTP404StatusCodeError')
+                continue
+            elseif strcmp(ME.identifier, 'NWB:BuildRepoDownloadUrl:UnsupportedRepository')
+                error('NWB:InstallExtension:UnsupportedRepository', ...
+                    ['Extension "%s" is located in an unsupported repository ', ...
+                     '/ source location. \nPlease create an issue on MatNWB''s ', ...
+                     'github page'], extensionName)
+            else
+                rethrow(ME)
+            end
+        end
+    end
+end

+matnwb/+extension/+internal/downloadZippedRepo.m

@@ -0,0 +1,37 @@
+function repoFolder = downloadZippedRepo(githubUrl, targetFolder)
+%downloadZippedRepo - Download a zipped repository
+
+    % Create a temporary path for storing the downloaded file.
+    [~, ~, fileType] = fileparts(githubUrl);
+    tempFilepath = [tempname, fileType];
+
+    % Download the file containing the zipped repository
+    tempFilepath = websave(tempFilepath, githubUrl);
+    fileCleanupObj = onCleanup( @(fname) delete(tempFilepath) );
+
+    unzippedFiles = unzip(tempFilepath, tempdir);
+    unzippedFolder = unzippedFiles{1};
+    if endsWith(unzippedFolder, filesep)
+        unzippedFolder = unzippedFolder(1:end-1);
+    end
+
+    [~, repoFolderName] = fileparts(unzippedFolder);
+    targetFolder = fullfile(targetFolder, repoFolderName);
+
+    if isfolder(targetFolder)
+        try
+            rmdir(targetFolder, 's')
+        catch
+            error('Could not delete previously downloaded extension which is located at:\n"%s"', targetFolder)
+        end
+    else
+        % pass
+    end
+
+    movefile(unzippedFolder, targetFolder);
+
+    % Delete the temp zip file
+    clear fileCleanupObj
+
+    repoFolder = targetFolder;
+end

+matnwb/+extension/getExtensionInfo.m

@@ -0,0 +1,49 @@
+function info = getExtensionInfo(extensionName)
+% getExtensionInfo - Get metadata for the specified Neurodata extension
+%
+% Syntax:
+%  info = matnwb.extension.GETEXTENSIONINFO(extensionName) Returns a struct 
+%  with metadata/information about the specified extension. The extension
+%  must be registered in the Neurodata Extension Catalog.
+%
+% Input Arguments:
+%  - extensionName (string) - 
+%    Name of a Neurodata Extension, e.g "ndx-miniscope".
+%
+% Output Arguments:
+%  - info (struct) - 
+%    Struct with metadata / information for the specified extension. The struct 
+%    has the following fields: 
+%    
+%    - name - The name of the extension.
+%    - version - The current version of the extension.
+%    - last_updated - A timestamp indicating when the extension was last updated.
+%    - src - The URL to the source repository or homepage of the extension.
+%    - license - The license type under which the extension is distributed.
+%    - maintainers - A cell array or array of strings listing the maintainers.
+%    - readme - A string containing the README documentation or description.
+%
+% Usage:
+%  Example 1 - Retrieve and display information for the 'ndx-miniscope' extension::
+% 
+%    info = matnwb.extension.getExtensionInfo('ndx-miniscope');
+% 
+%    % Display the version of the extension.
+%    fprintf('Extension version: %s\n', info.version);
+%
+% See also: 
+%   matnwb.extension.listExtensions
+
+    arguments
+        extensionName (1,1) string
+    end
+
+    T = matnwb.extension.listExtensions();
+    isMatch = T.name == extensionName;
+    extensionList = join( compose("  %s", [T.name]), newline );
+    assert( ...
+        any(isMatch), ...
+        'NWB:DisplayExtensionMetadata:ExtensionNotFound', ...
+        'Extension "%s" was not found in the extension catalog:\n%s', extensionName, extensionList)
+    info = table2struct(T(isMatch, :));
+end

+matnwb/+extension/installAll.m

@@ -0,0 +1,6 @@
+function installAll()
+    T = matnwb.extension.listExtensions();
+    for i = 1:height(T)
+        matnwb.extension.installExtension( T.name(i) )
+    end
+end

+matnwb/+extension/installExtension.m

@@ -0,0 +1,49 @@
+function installExtension(extensionName, options)
+% installExtension - Install NWB extension from Neurodata Extensions Catalog
+%
+%   matnwb.extension.nwbInstallExtension(extensionName) installs a Neurodata 
+%   Without Borders (NWB) extension from the Neurodata Extensions Catalog to 
+%   extend the functionality of the core NWB schemas. 
+
+    arguments
+        extensionName (1,1) string
+        options.savedir (1,1) string = misc.getMatnwbDir()
+    end
+
+    import matnwb.extension.internal.downloadExtensionRepository
+
+    repoTargetFolder = fullfile(userpath, "NWB-Extension-Source");
+    if ~isfolder(repoTargetFolder); mkdir(repoTargetFolder); end
+
+    T = matnwb.extension.listExtensions();
+    isMatch = T.name == extensionName;
+
+    extensionList = join( compose("  %s", [T.name]), newline );
+    assert( ...
+        any(isMatch), ...
+        'NWB:InstallExtension:ExtensionNotFound', ...
+        'Extension "%s" was not found in the extension catalog:\n', extensionList)
+
+    repositoryUrl = T{isMatch, 'src'};
+
+    [wasDownloaded, repoTargetFolder] = ...
+        downloadExtensionRepository(repositoryUrl, repoTargetFolder, extensionName);
+
+    if ~wasDownloaded
+        error('NWB:InstallExtension:DownloadFailed', ...
+            'Failed to download spec for extension "%s"', extensionName)
+    end
+    L = dir(fullfile(repoTargetFolder, 'spec', '*namespace.yaml'));
+    assert(...
+        ~isempty(L), ...
+        'NWB:InstallExtension:NamespaceNotFound', ...
+        'No namespace file was found for extension "%s"', extensionName ...
+        )
+    assert(...
+        numel(L)==1, ...
+        'NWB:InstallExtension:MultipleNamespacesFound', ...
+        'More than one namespace file was found for extension "%s"', extensionName ...
+        )
+    generateExtension( fullfile(L.folder, L.name), 'savedir', options.savedir );
+    fprintf("Installed extension ""%s"".\n", extensionName)
+end

+matnwb/+extension/listExtensions.m

@@ -0,0 +1,83 @@
+function extensionTable = listExtensions(options)
+% listExtensions - List available extensions in the Neurodata Extension Catalog
+%
+% Syntax:
+%  extensionTable = matnwb.extension.LISTEXTENSIONS() returns a table where
+%  each row holds information about a registered extension.
+%
+% Output Arguments:  
+%  - extensionTable (table) - 
+%    Table of metadata / information for each registered extension. The table 
+%    has the following columns:
+%  
+%    - name - The name of the extension.
+%    - version - The current version of the extension.
+%    - last_updated - A timestamp indicating when the extension was last updated.
+%    - src - The URL to the source repository or homepage of the extension.
+%    - license - The license type under which the extension is distributed.
+%    - maintainers - A cell array or array of strings listing the maintainers.
+%    - readme - A string containing the README documentation or description.
+%
+% Usage:
+%  Example 1 - List and display extensions::
+% 
+%    T = matnwb.extension.listExtensions();
+%    disp(T)
+%
+% See also: 
+%   matnwb.extension.getExtensionInfo
+
+    arguments
+        % Refresh - Flag to refresh the catalog (Only relevant if the
+        % remote catalog has been updated).
+        options.Refresh (1,1) logical = false
+    end
+
+    persistent extensionRecords
+
+    if isempty(extensionRecords) || options.Refresh
+        catalogUrl = "https://raw.githubusercontent.com/nwb-extensions/nwb-extensions.github.io/refs/heads/main/data/records.json";
+        extensionRecords = jsondecode(webread(catalogUrl));
+        extensionRecords = consolidateStruct(extensionRecords);
+
+        extensionRecords = struct2table(extensionRecords);
+
+        fieldsKeep = ["name", "version", "last_updated", "src", "license", "maintainers", "readme"];
+        extensionRecords = extensionRecords(:, fieldsKeep);
+
+        for name = fieldsKeep
+            if ischar(extensionRecords.(name){1})
+                extensionRecords.(name) = string(extensionRecords.(name));
+            end
+        end
+    end
+    extensionTable = extensionRecords;
+end
+
+function structArray = consolidateStruct(S)
+    % Get all field names of S
+    mainFields = fieldnames(S);
+
+    % Initialize an empty struct array
+    structArray = struct();
+
+    % Iterate over each field of S
+    for i = 1:numel(mainFields)
+        subStruct = S.(mainFields{i}); % Extract sub-struct
+
+        % Add all fields of the sub-struct to the struct array
+        fields = fieldnames(subStruct);
+        for j = 1:numel(fields)
+            structArray(i).(fields{j}) = subStruct.(fields{j});
+        end
+    end
+
+    % Ensure consistency by filling missing fields with []
+    allFields = unique([fieldnames(structArray)]);
+    for i = 1:numel(structArray)
+        missingFields = setdiff(allFields, fieldnames(structArray(i)));
+        for j = 1:numel(missingFields)
+            structArray(i).(missingFields{j}) = [];
+        end
+    end
+end

+tests/+unit/InstallExtensionTest.m

@@ -0,0 +1,87 @@
+classdef InstallExtensionTest < matlab.unittest.TestCase
+
+    methods (TestClassSetup)
+        function setupClass(testCase)
+            % Get the root path of the matnwb repository
+            rootPath = misc.getMatnwbDir();
+
+            % Use a fixture to add the folder to the search path
+            testCase.applyFixture(matlab.unittest.fixtures.PathFixture(rootPath));
+
+            % Use a fixture to create a temporary working directory
+            testCase.applyFixture(matlab.unittest.fixtures.WorkingFolderFixture);
+            generateCore('savedir', '.');
+        end
+    end
+
+    methods (Test)
+        function testInstallExtensionFailsWithNoInputArgument(testCase)
+            testCase.verifyError(...
+                @(varargin) nwbInstallExtension(), ...
+                'NWB:InstallExtension:MissingArgument')
+        end
+
+        function testInstallExtension(testCase)
+            nwbInstallExtension("ndx-miniscope", 'savedir', '.')
+
+            testCase.verifyTrue(isfolder('./+types/+ndx_miniscope'), ...
+                'Folder with extension types does not exist')
+        end
+
+        function testUseInstalledExtension(testCase)
+            nwbObject = testCase.initNwbFile();
+
+            miniscopeDevice = types.ndx_miniscope.Miniscope(...
+                'deviceType', 'test_device', ...
+                'compression', 'GREY', ...
+                'frameRate', '30fps', ...
+                'framesPerFile', int8(100) );
+
+            nwbObject.general_devices.set('TestMiniscope', miniscopeDevice);
+
+            testCase.verifyClass(nwbObject.general_devices.get('TestMiniscope'), ...
+                'types.ndx_miniscope.Miniscope')
+        end
+
+        function testGetExtensionInfo(testCase)
+            extensionName = "ndx-miniscope";
+            metadata = matnwb.extension.getExtensionInfo(extensionName);
+            testCase.verifyClass(metadata, 'struct')
+            testCase.verifyEqual(metadata.name, extensionName)
+        end
+
+        function testDownloadUnknownRepository(testCase)
+            repositoryUrl = "https://www.unknown-repo.com/anon/my_nwb_extension";
+            testCase.verifyError(...
+                @() matnwb.extension.internal.downloadExtensionRepository(repositoryUrl, "", "my_nwb_extension"), ...
+                 'NWB:InstallExtension:UnsupportedRepository');
+        end
+
+        function testBuildRepoDownloadUrl(testCase)
+
+            import matnwb.extension.internal.buildRepoDownloadUrl
+
+            repoUrl = buildRepoDownloadUrl('https://github.com/user/test', 'main');
+            testCase.verifyEqual(repoUrl, 'https://github.com/user/test/archive/refs/heads/main.zip')
+
+            repoUrl = buildRepoDownloadUrl('https://github.com/user/test/', 'main');
+            testCase.verifyEqual(repoUrl, 'https://github.com/user/test/archive/refs/heads/main.zip')
+
+            repoUrl = buildRepoDownloadUrl('https://gitlab.com/user/test', 'main');
+            testCase.verifyEqual(repoUrl, 'https://gitlab.com/user/test/-/archive/main/test-main.zip')
+
+            testCase.verifyError(...
+                @() buildRepoDownloadUrl('https://unsupported.com/user/test', 'main'), ...
+                'NWB:BuildRepoDownloadUrl:UnsupportedRepository')
+        end
+    end
+
+    methods (Static)
+        function nwb = initNwbFile()
+            nwb = NwbFile( ...
+                'session_description', 'test file for nwb extension', ...
+                'identifier', 'export_test', ...
+                'session_start_time', datetime("now", 'TimeZone', 'local') );
+        end
+    end
+end