From 41de2564d0fbdcddbd4bc5889af2fd78b4a0295d Mon Sep 17 00:00:00 2001
From: Vincent <vp35@duke.edu>
Date: Sat, 19 Sep 2020 17:16:58 -0400
Subject: [PATCH 01/22] Make full use of the .prm file in kilosort import

prevents overwriting of prm parameter values by default values
fixes renumbering of spike sites when KiloSort returns partial channel map
---
 +jrclust/+import/kilosort.m | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/+jrclust/+import/kilosort.m b/+jrclust/+import/kilosort.m
index a4e4ac7a..b553ae1f 100644
--- a/+jrclust/+import/kilosort.m
+++ b/+jrclust/+import/kilosort.m
@@ -20,22 +20,23 @@
 channelMap = phyData.channel_map + 1;
 channelPositions = phyData.channel_positions;
 
-cfgData.sampleRate = params.sample_rate;
-cfgData.nChans = params.n_channels_dat;
-cfgData.dataType = params.dtype;
-cfgData.headerOffset = params.offset;
-cfgData.siteMap = channelMap;
-cfgData.siteLoc = channelPositions;
-cfgData.shankMap = ones(size(channelMap), 'like', channelMap); % this can change with a prm file
-cfgData.rawRecordings = {params.dat_path};
-
 % check for existence of .prm file. if exists use it as a template.
 [a,b,~] = fileparts(params.dat_path);
 prm_path = [a,filesep,b,'.prm'];
 if exist(prm_path,'file')
     cfgData.template_file = prm_path;
+else
+    cfgData.sampleRate = params.sample_rate;
+    cfgData.nChans = params.n_channels_dat;
+    cfgData.dataType = params.dtype;
+    cfgData.headerOffset = params.offset;
+    cfgData.siteMap = channelMap;
+    cfgData.siteLoc = channelPositions;
+    cfgData.shankMap = ones(size(channelMap), 'like', channelMap);
+    cfgData.rawRecordings = {params.dat_path};
 end
 hCfg = jrclust.Config(cfgData);
+siteMapIndex = find(ismember(hCfg.siteMap,channelMap)); % KS may have returned a partial channel map, i.e. ~all(ismember(hCfg.siteMap,channelMap)) 
 
 % load spike data
 amplitudes = phyData.amplitudes;
@@ -62,9 +63,8 @@
 spikeSites = zeros(size(spikeClusters), 'like', spikeClusters);
 for iTemplate = 1:nTemplates
     template = squeeze(templates(iTemplate, :, :));
-    [~, tSite] = min(min(template));
-
-    spikeSites(spikeTemplates == iTemplate) = tSite;
+    [~, tSite] = max(max(abs(template)));
+    spikeSites(spikeTemplates == iTemplate) = siteMapIndex(tSite); %setting spikeSites as tSites directly would renumber channel map in case KS has returned a partial channel map 
 end
 
 %%% try to detect the recording file

From fa6e381ae6c1ff7805d687cefcfe07c1850232cc Mon Sep 17 00:00:00 2001
From: Vincent <vp35@duke.edu>
Date: Sat, 19 Sep 2020 17:18:57 -0400
Subject: [PATCH 02/22] adds trial file and PSTH parameters during bootstrap if
 file is present

---
 @JRC/bootstrap.m | 41 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 41 insertions(+)

diff --git a/@JRC/bootstrap.m b/@JRC/bootstrap.m
index e052c00b..7aaf02b9 100644
--- a/@JRC/bootstrap.m
+++ b/@JRC/bootstrap.m
@@ -279,6 +279,15 @@ function bootstrap(obj, varargin)
     end
 
     cfgData.probe_file = probeFile;
+    
+    % check whether there's a trial file in the directory as well
+    [cfgData.trialFile,cfgData.psthTimeLimits] = getTrialFile(cfgData.outputDir);
+    if ~isempty(cfgData.trialFile)
+        % check whether the .meta specifies PSTH display parameters
+        if isfield(SMeta,'psthTimeLimits'); cfgData.psthTimeLimits=str2num(SMeta.psthTimeLimits); end
+        if isfield(SMeta,'psthTimeBin'); cfgData.psthTimeBin=SMeta.psthTimeBin; end
+        if isfield(SMeta,'psthXTick'); cfgData.psthXTick=SMeta.psthXTick; end
+    end
 end
 
 function probeFile = probeFileInDir(workingdir)
@@ -322,6 +331,38 @@ function bootstrap(obj, varargin)
         end
     end
 end
+
+function [trialFile,psthTimeLimits] = getTrialFile(workingdir, ask)
+    if nargin < 2
+        ask = 0;
+    end
+
+    trialFile = trialFileInDir(workingdir);
+
+    if ~isempty(trialFile) && ask % found a trial file in working directory; confirm
+        dlgAns = questdlg(sprintf('Found trial file ''%s''. Use it?', trialFile));
+        if strcmp(dlgAns, 'Yes')
+            return;
+        else
+            trialFile = '';
+        end
+    end
+    
+    psthTimeLimits = [-0.1 0.1]; % default time range (in s) over which to display PSTH        
+end
+
+
+function trialFile = trialFileInDir(workingdir)
+   trialFile = '';
+
+    d = dir(fullfile(workingdir, '*trial.*'));
+    if isempty(d)
+        return;
+    end
+
+    trialFile = fullfile(workingdir, d(1).name);
+end
+
 % function hCfg = bootstrapGUI() % WIP
 %     %BOOTSTRAPGUI Show all (common) parameters
 %     % load old2new param set and convert to new2old

From 32c08fe1abfc46edbbcc58ae4c88c570e9a4c016 Mon Sep 17 00:00:00 2001
From: Vincent <vp35@duke.edu>
Date: Sat, 24 Oct 2020 11:46:00 -0400
Subject: [PATCH 03/22] temp solution for saveRes

---
 @JRC/processArgs.m | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/@JRC/processArgs.m b/@JRC/processArgs.m
index 4f828701..6995b96f 100644
--- a/@JRC/processArgs.m
+++ b/@JRC/processArgs.m
@@ -153,7 +153,7 @@ function processArgs(obj)
                 obj.hCfg = hCfg_;
                 obj.res = res_;
 
-                obj.saveRes();
+                obj.saveRes(~obj.args{2});
                 obj.hCfg.save('', 1);
 
                 obj.isCompleted = 1;

From 92422ec190d8357ed37a31b035d53468c2cd728f Mon Sep 17 00:00:00 2001
From: Steve Van Hooser <vanhoosr@brandeis.edu>
Date: Tue, 27 Oct 2020 20:12:52 -0400
Subject: [PATCH 04/22] added bootstrapNDI, ndiRecording object, both seem to
 work well

---
 +jrclust/+detect/ndiRecording.m | 101 +++++++++++++++++++++++++++++
 +jrclust/+detect/newRecording.m |   3 +-
 @JRC/bootstrap.m                |   4 ++
 @JRC/bootstrapNDI.m             | 108 ++++++++++++++++++++++++++++++++
 json/params.json                |   3 +-
 5 files changed, 217 insertions(+), 2 deletions(-)
 create mode 100644 +jrclust/+detect/ndiRecording.m
 create mode 100644 @JRC/bootstrapNDI.m

diff --git a/+jrclust/+detect/ndiRecording.m b/+jrclust/+detect/ndiRecording.m
new file mode 100644
index 00000000..c0253beb
--- /dev/null
+++ b/+jrclust/+detect/ndiRecording.m
@@ -0,0 +1,101 @@
+classdef ndiRecording < jrclust.interfaces.RawRecording
+    %NDIRECORDING Model of an element in NDI
+    %% NDI-SPECIFIC PROPERTIES
+
+    properties (GetAccess=public,SetAccess=private)
+        S; % ndi.session.dir that holds the experimental session data
+        E; % the ndi.element object that holds the data
+        epoch_id; % the epoch ids that are included
+        nsamples; % the number of samples per epoch
+        t0_t1; % start and end times, in local device time, of each epoch
+    end;
+
+    %% LIFECYCLE    
+    methods
+        function obj = ndiRecording(epochname, hCfg)
+            %NDIRECORDING Construct an instance of this class
+            % set object data type
+            obj = obj@jrclust.interfaces.RawRecording(epochname, hCfg);
+            obj.S = ndi.session.dir(hCfg.ndiPath);
+            E = getelements(obj.S,'element.name',hCfg.ndiElementName,'element.reference',hCfg.ndiElementReference);
+            if iscell(E) & numel(E)==1,
+                obj.E = E{1};
+            else,
+                obj.isError = 1;
+                obj.errMsg = 'No such element found.';
+                return;
+            end;
+
+            try
+                obj.hCfg = hCfg;
+            catch ME
+                obj.errMsg = ME.message;
+                obj.isError = 1;
+                return;
+            end
+
+            et = epochtable(obj.E);
+            
+            [parentdir,epoch_id] = fileparts(epochname);
+            match = find(strcmp(epoch_id,{et.epoch_id}));
+            if isempty(match),
+                 obj.isError = 1;
+                 obj.errMsg = ['No such epoch ' et_here '.'];
+                 return;
+            end;
+            obj.epoch_id = et(match).epoch_id;
+            % find clock that is dev local time
+            foundMatch = 0;
+            for i=1:numel(et(match).epoch_clock),
+                 if strcmp(et(match).epoch_clock{i}.type,'dev_local_time'),
+                     foundMatch = i;
+                     break;
+                 end;
+            end;
+            if foundMatch,
+                obj.t0_t1 = et(match).t0_t1{foundMatch};
+            else,
+                error(['No clock type ''dev_local_time''.']);
+            end;
+            obj.nsamples = 1+diff(times2samples(obj.E,epoch_id,obj.t0_t1));
+            obj.dshape = [hCfg.nChans, obj.nsamples];
+        end % ndiRecording()
+
+        function openRaw(obj)
+            %OPENRAW Open the raw recording file for reading - does nothing for ndiRecording
+            obj.rawIsOpen = 1;
+        end
+        function closeRaw(obj)
+            %CLOSERAW Close the raw recording file for reading - does nothing for ndiRecording
+            obj.rawIsOpen = 0;
+        end
+
+        function roi = readRawROI(obj, rows, cols)
+            %READRAWROI Get a region of interest by rows/cols from the raw file
+            t0t1 = samples2times(obj.E, obj.epoch_id, cols([1 end]));
+            roi = readtimeseries(obj.E, obj.epoch_id, t0t1(1), t0t1(2));
+            roi = roi'; % switch to column-based samples
+            roi = roi(rows,:); % if only a subset requested, return only the subset
+        end % readRawROI()
+
+
+    end % methods
+
+    %% GETTERS/SETTERS
+    methods
+        function set.S(obj, val)
+            if ~isa(val,'ndi.session.dir'),
+                 error(['S must be of type ndi.session.dir']);
+            end;
+            obj.S = val;
+        end
+        function set.E(obj,val)
+            if ~isa(val,'ndi.element') & ~isa(val,'ndi.time.timeseries'),
+                 error(['E must be an ndi.element and an ndi.time.timeseries']);
+            end;
+            obj.E = val;
+        end;
+	
+    end
+end
+
diff --git a/+jrclust/+detect/newRecording.m b/+jrclust/+detect/newRecording.m
index 147fae20..d6b7c075 100644
--- a/+jrclust/+detect/newRecording.m
+++ b/+jrclust/+detect/newRecording.m
@@ -3,7 +3,8 @@
     switch hCfg.recordingFormat
         case 'Intan'
             hRec = jrclust.detect.IntanRecording(rawPath, hCfg);
-
+	case 'ndi',
+            hRec = jrclust.detect.ndiRecording(rawPath, hCfg);
         otherwise % SpikeGLX .bin/.dat
             hRec = jrclust.detect.Recording(rawPath, hCfg);
 
diff --git a/@JRC/bootstrap.m b/@JRC/bootstrap.m
index 18911d75..709fc435 100644
--- a/@JRC/bootstrap.m
+++ b/@JRC/bootstrap.m
@@ -2,6 +2,10 @@ function bootstrap(obj, varargin)
     %BOOTSTRAP Bootstrap a JRCLUST session
     %   metafile: optional string; path (or glob) to meta file(s)
     if nargin > 1
+	if strcmpi(varargin{1},'ndi'),
+		obj.bootstrapNDI(varargin{2:end});
+		return;
+	end;
         metafile_ = jrclust.utils.absPath(varargin{1});
         if isempty(metafile_) % TODO: warn?
             metafile = '';
diff --git a/@JRC/bootstrapNDI.m b/@JRC/bootstrapNDI.m
new file mode 100644
index 00000000..131d0a96
--- /dev/null
+++ b/@JRC/bootstrapNDI.m
@@ -0,0 +1,108 @@
+function bootstrapNDI(obj, varargin)
+    %BOOTSTRAPNDI Build a configuration PRM file for reading an ndi.element.timeseries object
+    %
+    % BOOTSTRAPNDI(OBJ, NDI_SESSION_OBJ, NDI_ELEMENT_TIMESERIES_OBJ ...])
+    %
+    % Generates a PRM file for the ndi.element.timeseries object specified that is part of
+    % the ndi.session that is indicated. 
+    %
+    % The JRCLUST analysis files will be installed at the path to NDI_SESSION_DIR_OBJ in a folder
+    % called '.JRCLUST' and another subfolder with the string name of NDI_ELEMENT_TIMESERIES_OBJ. 
+    % 
+    % During bootstrap, all epochs of NDI_ELEMENT_TIMESERIES_OBJ are added for extraction. They
+    % can be edited down as needed in the editor.
+    % 
+    % Example:
+    % 
+    % 
+
+    if numel(varargin)~=2,
+       error(['Usage: jrc(''bootstrap'',''ndi'',ndi_session_dir_obj,ndi_element_timeseries_obj)']);
+    end;
+
+    % Step 1: check our arguments and set our variables
+    %   S - the ndi.session.dir
+    %   E - the ndi.element.timeseries
+    %   epoch_ids - the epoch ids
+
+    try,
+       ndi_v = ndi.version();
+    catch,
+       error(['Could not find ndi. Need to install https://github.com/VH-Lab/NDI-matlab ']);
+    end;
+
+    % confirm inputs
+    try,
+       S = varargin{1}; 
+       if ~isa(S,'ndi.session.dir'),
+           error(['Input argument that follows NDI is not of type ndi.session.dir']);
+       end;
+       E = varargin{2};
+       if ~isa(E,'ndi.element') | ~isa(E,'ndi.time.timeseries'),
+           error(['Input that follows ndi.session is not of type ndi.element.timeseries.']);
+       end;
+       if E.session~=S,
+           error(['The ndi.element.timeseries must be part of the ndi.session provided.']);
+       end;
+       et = E.epochtable();
+       epoch_ids = {et.epoch_id};
+    catch ME,
+       disp(['Usage: jrc(''bootstrap'',''ndi'',ndi_session_dir_obj,ndi_element_timeseries_obj)']);
+       error(ME.message);
+    end;
+
+    Estring = E.elementstring();
+    Estring(find(Estring==' ')) = '_';
+    output_dir = [S.path() filesep '.JRCLUST' filesep Estring];
+    output_file = [output_dir filesep 'jrclust.prm'];
+    if ~exist(output_dir,'dir'),
+        try,
+           mkdir(output_dir);
+        end;
+    end;
+
+    cfgData = struct();
+
+    [data,t,timeref] = E.readtimeseries(1,0,0); % read 1 sample
+    cfgData.tallSkinny = 1; % we will transpose later so this is met
+    cfgData.outputDir = output_dir;
+
+    cfgData.headerOffset = 0; % does not make sense for NDI
+    cfgData.nChans = size(data,2);
+    cfgData.sampleRate = E.samplerate(1); % get sample rate of first epoch; assume it's the same
+    cfgData.rawRecordings = epoch_ids;
+    
+    cfgData.dataTypeRaw = 'int16'; % meaningless
+    cfgData.dataTypeExtracted = 'single'; %  we will convert to single resolution
+    cfgData.bitScaling = 1;
+    cfgData.recordingFormat = 'ndi';
+    cfgData.ndiPath = S.path();
+    cfgData.ndiElementName = E.name;
+    cfgData.ndiElementReference = E.reference;
+
+    q = ndi.query('','depends_on',E.id(),'') & ndi.query('','isa','probe file','');
+    docs = S.database_search(q);
+    if ~isempty(docs),
+         error(['I do not know how to do this yet.']);
+         % set probe parameters from doc
+    else,
+            % set some trivial defaults
+            cfgData.siteMap = 1:cfgData.nChans;
+            cfgData.siteLoc = [zeros(cfgData.nChans, 1), 50*(0:cfgData.nChans-1)'];
+            cfgData.shankMap = ones(cfgData.nChans, 1);
+    end
+
+    % construct the Config object from specified data
+    hCfg_ = jrclust.Config(cfgData);
+
+    configFile = fullfile(hCfg_.outputDir, ['jrclust', '.prm']);
+    if ~exist(configFile,'file'),
+        fclose(fopen(configFile,'w')); % touch the file and close it
+    end;
+    hCfg_.setConfigFile(configFile, 0);
+    hCfg_.save('', 1);
+
+    obj.hCfg = hCfg_;
+    obj.hCfg.edit();
+end
+
diff --git a/json/params.json b/json/params.json
index 044f9381..4122088b 100644
--- a/json/params.json
+++ b/json/params.json
@@ -562,7 +562,8 @@
                 "attributes": ["scalartext"],
                 "values": [
                     "SpikeGLX",
-                    "Intan"
+                    "Intan",
+                    "ndi"
                 ]
             },
             "section": ["recording file"]

From 1ba0024959f86a4269b51127d61d7d40270ff4a5 Mon Sep 17 00:00:00 2001
From: Steve Van Hooser <vanhoosr@brandeis.edu>
Date: Wed, 28 Oct 2020 20:53:00 -0400
Subject: [PATCH 05/22] detection, bootstrapping working, partially tested

---
 +jrclust/+detect/@DetectController/findSecondaryPeaks.m | 4 ++--
 +jrclust/+detect/ndiRecording.m                         | 2 +-
 +jrclust/+views/@TracesController/TracesController.m    | 1 +
 @JRC/bootstrapNDI.m                                     | 4 ++--
 4 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/+jrclust/+detect/@DetectController/findSecondaryPeaks.m b/+jrclust/+detect/@DetectController/findSecondaryPeaks.m
index 4ed4fd44..2da30a33 100644
--- a/+jrclust/+detect/@DetectController/findSecondaryPeaks.m
+++ b/+jrclust/+detect/@DetectController/findSecondaryPeaks.m
@@ -1,7 +1,7 @@
 function [spikeSites2, spikeSites3] = findSecondaryPeaks(obj, spikeWindows, spikeSites)
     %FINDSECONDARYPEAKS Find secondary peaks
     %   spikeWindows: nSamples x nSites x nSpikes
-    evtNeighbors = 2:obj.hCfg.nSitesEvt; % sites within merge radius, excluding ref sites
+    evtNeighbors = 2:obj.hCfg.nSitesEvt; % sites within merge radius, excluding ref sites 
     siteNeighbors2 = obj.hCfg.siteNeighbors(evtNeighbors, spikeSites);
 
     spikeWindows2 = spikeWindows(:, evtNeighbors, :);
@@ -22,4 +22,4 @@
     else
         spikeSites3 = [];
     end
-end
\ No newline at end of file
+end
diff --git a/+jrclust/+detect/ndiRecording.m b/+jrclust/+detect/ndiRecording.m
index c0253beb..1bc12944 100644
--- a/+jrclust/+detect/ndiRecording.m
+++ b/+jrclust/+detect/ndiRecording.m
@@ -75,7 +75,7 @@ function closeRaw(obj)
             t0t1 = samples2times(obj.E, obj.epoch_id, cols([1 end]));
             roi = readtimeseries(obj.E, obj.epoch_id, t0t1(1), t0t1(2));
             roi = roi'; % switch to column-based samples
-            roi = roi(rows,:); % if only a subset requested, return only the subset
+            roi = single(roi(rows,:)); % if only a subset requested, return only the subset
         end % readRawROI()
 
 
diff --git a/+jrclust/+views/@TracesController/TracesController.m b/+jrclust/+views/@TracesController/TracesController.m
index 3d2e0fcf..1b1b1663 100644
--- a/+jrclust/+views/@TracesController/TracesController.m
+++ b/+jrclust/+views/@TracesController/TracesController.m
@@ -92,6 +92,7 @@ function keyPressFigTraces(obj, ~, hEvent)
 
                     tracesRaw_ = cellfun(@(lims) obj.hRec.readRawROI(obj.hCfg.siteMap, lims(1):lims(2)), multiBounds, 'UniformOutput', 0);
                     obj.tracesRaw = jrclust.utils.neCell2mat(tracesRaw_);
+                    %if size(tracesRaw_{1},2)~=diff(multiBounds{1})+1, keyboard; end; % sometimes a useful debugging line
 
                     obj.tracesRaw = u2i(obj.tracesRaw);
                     obj.hFigTraces.figData.windowBounds = windowBounds;
diff --git a/@JRC/bootstrapNDI.m b/@JRC/bootstrapNDI.m
index 131d0a96..732dc34b 100644
--- a/@JRC/bootstrapNDI.m
+++ b/@JRC/bootstrapNDI.m
@@ -53,7 +53,7 @@ function bootstrapNDI(obj, varargin)
 
     Estring = E.elementstring();
     Estring(find(Estring==' ')) = '_';
-    output_dir = [S.path() filesep '.JRCLUST' filesep Estring];
+    output_dir = [S.path filesep '.JRCLUST' filesep Estring];
     output_file = [output_dir filesep 'jrclust.prm'];
     if ~exist(output_dir,'dir'),
         try,
@@ -76,7 +76,7 @@ function bootstrapNDI(obj, varargin)
     cfgData.dataTypeExtracted = 'single'; %  we will convert to single resolution
     cfgData.bitScaling = 1;
     cfgData.recordingFormat = 'ndi';
-    cfgData.ndiPath = S.path();
+    cfgData.ndiPath = S.path;
     cfgData.ndiElementName = E.name;
     cfgData.ndiElementReference = E.reference;
 

From f61a62a9785d9d1e23e31747b5ebba8823a388e9 Mon Sep 17 00:00:00 2001
From: Steve Van Hooser <vanhoosr@brandeis.edu>
Date: Wed, 4 Nov 2020 20:25:18 -0500
Subject: [PATCH 06/22] ndi update almost done

---
 +jrclust/+export/ndi.m                     | 190 ++++++++
 +jrclust/+utils/+DataHash/DataHash.m       | 534 +++++++++++++++++++++
 +jrclust/+utils/+DataHash/license.txt      |  25 +
 +jrclust/+utils/+DataHash/uTest_DataHash.m | 367 ++++++++++++++
 +jrclust/+utils/assign.m                   |  35 ++
 5 files changed, 1151 insertions(+)
 create mode 100644 +jrclust/+export/ndi.m
 create mode 100644 +jrclust/+utils/+DataHash/DataHash.m
 create mode 100644 +jrclust/+utils/+DataHash/license.txt
 create mode 100644 +jrclust/+utils/+DataHash/uTest_DataHash.m
 create mode 100644 +jrclust/+utils/assign.m

diff --git a/+jrclust/+export/ndi.m b/+jrclust/+export/ndi.m
new file mode 100644
index 00000000..fdb12852
--- /dev/null
+++ b/+jrclust/+export/ndi.m
@@ -0,0 +1,190 @@
+function ndi(hCfg, varargin)
+    % NDI - export spike clusters to NDI
+    % 
+    % NDI(HCFG, ...)
+    %
+    % Given a JRCLUST/Config object, export sorted spike clusters to NDI.
+    %
+    % The NDI session and NDI element are specified in the configuration information:
+    % hCfg.ndiPath is the path to the NDI session to be modified.
+    % hCfg.ElementName is the element name, and hCfg.ElementReference is the element reference.
+    % The new neurons will be named [hCfg.ElementName '_N'], where N is the cluster number.
+    % 
+    % An NDI document of type JRCLUST_parameters is created and added to the NDI session.
+    % It depends on the NDI element being sorted, and has a name equal to the JRCLUST parameter file name
+    % (usually just 'jrclust.prm' unless the user has changed it). Its binary component is the text of the
+    % JRCLUST parameter file. It also has a field JRCLUST.checksum that has the MD5 checksum of the
+    % the jrclust_res.mat file. This checksum is used to check if the clustering has changed since a previous
+    % export.
+    %
+    % If the clusters have not changed since a previous writing, no action is taken unless
+    % the 'force' parameter is set to 1. If the clusters have changed, then the user is asked to
+    % confirm that he/she wants to delete the previous cluster document (and the neurons that depend on
+    % it) before re-writing.
+    % 
+    % This function takes additional name/value pairs that modify its operation:
+    %
+    % Parameter (default)    | Description
+    % ----------------------------------------------------------------
+    % interactive (1)        | Ask before deleting existing clusters
+    % interactviakey         | Interact via keyboard (otherwise, use dialog)
+    %   (hCfg.batch)         |    
+    % forceReplace (0)       | Re-write clusters even if they have not changed
+    %                        |   (will delete any documents that depend on the neurons,
+    %                        |   including analyses.)
+    %
+
+        % Step 1a: Establish default parameters:
+
+    interactive = 1;
+    interactiveviakey = hCfg.batchMode;
+    forceReplace = 0;
+
+        % Step 1b: Update parameters with any user requests
+    jrclust.utils.assign(varargin{:});
+
+        % Step 1c: Check that we have NDI
+    try,
+        ndi_v = ndi.version();
+    catch,
+        error(['Could not determine NDI version. Check that https://github.com/VH-Lab/NDI-matlab is installed.']);
+    end;
+
+       % Step 2: load NDI session and element and check for errors
+ 
+    S = ndi.session.dir(hCfg.ndiPath);
+    E = getelements(S,'element.name',hCfg.ndiElementName,'element.reference',hCfg.ndiElementReference);
+    if iscell(E) & numel(E)==1,
+        E = E{1};
+    else,
+        error(['No such element ' nCfg.ndiElementName ' with reference ' num2str(hCfg.ndiElementReference) ' found.']);
+    end;
+
+       % Step 3: check for existing cluster document; if it exists, ask if we want to delete it
+
+    md5_value = jrclust.utils.DataHash.DataHash(hCfg.resFile,'file','MD5');
+
+    [configPath, configFileName, configFileExt] = fileparts(hCfg.configFile);
+    configFileName = [configFileName configFileExt];
+   
+    q = ndi.query('','depends_on','element_id',E.id()) & ...
+       ndi.query('','isa','jrclust_clusters','') & ...
+       ndi.query('ndi_document.name','exact_string', configFileName,'');
+    doc = S.database_search(q);
+    if isempty(doc), % no docs,
+        % nothing to do
+    elseif numel(doc)>1,
+        % somehow we have more than one of these documents; this should not happen
+        % we will gracefully fix it
+        % 
+        fixDuplicates = 1;
+        if interactive,
+            if interactiveviakey,
+		disp(['We found more than 1 cluster document that matches this element and JRCLUST run (we found ' int2str(numel(doc)) ').']);
+                r = input(['Shall we delete them all? (Cannot continue if not yes.) [Y/n] :']);
+                if ~strcmpi(r,'Y') | ~strcmpi(r,'yes'),
+                    fixDuplicates = 0;
+                end;
+            else,
+                ButtonName = questdlg(['We found more than 1 cluster document that matches this element and JRCLUST run ' ...
+                                       '(we found (' int2str(numel(doc)) '). ' ...
+                                       'Shall we delete them all? (Cannot continue if not yes)'],'Delete extra cluster documents?',...
+                                       'Yes','No/Cancel','Yes');
+                if ~strcmp(ButtonName,'Yes'),
+                    fixDuplicates = 0;
+                end;
+            end;
+         end;
+         if fixDuplicates,
+            S.database_rm(doc);
+            doc = {};
+         else,
+             error(['We cannot continue with erroneous duplicate documents.']);
+         end;
+    else, % we have 1 doc
+        % see if it is up to date
+        up2date = strcmp(doc{1}.document_properties.jrclust_clusters.res_mat_MD5_checksum,md5_value);
+        if up2date,
+            if ~forceReplace, 
+               disp('we are already up to date, exiting...')
+               return;
+            else,
+               S.database_rm(doc);
+               doc = {};
+            end;
+        else,
+            replaceIt = 1; % default will be to replace
+            if interactive,
+                if interactiveviakey,
+                    r = input(['Update clusters to newer version held by JRCLUST? (Y/N): ']);
+                    if ~strcmpi(r,'Y') | ~strcmpi(r,'yes'),
+                        replaceIt = 0;
+                    end;
+                else,
+                    ButtonName = questdlg(['Update clusters to newer version held by JRCLUST?'],...
+                                       'Update?', 'Yes','No/Cancel','Yes');
+                    if ~strcmp(ButtonName,'Yes'),
+                        replaceIt = 0;
+                    end;
+                end;
+            end;
+            if replaceIt, 
+              S.database_rm(doc);
+              doc = {};
+            else, 
+               return; % nothing to do
+            end;
+         end;
+     end;
+    
+     % Step 4: If we are continuing, write the cluster document
+
+     d = ndi.document('apps/JRCLUST/jrclust_clusters.json', ...
+         'ndi_document.name', configFileName, ...
+         'jrclust_clusters.res_mat_MD5_checksum', md5_value);
+     d = d.set_dependency_value('element_id',E.id());
+     S.database_add(d);
+
+     % Step 5: Write each neuron; each should depend on the cluster document
+
+     % Step 5a: load the spike definitions and make sure they are annotated
+     c = load(hCfg.resFile);
+     if ~isfield(c,'clusterNotes'),
+         error(['Spikes have not yet been annotated. Annotate spikes first.']);
+     end;
+
+     % Step 5b: determine the mapping between samples and the NDI element's epoch
+
+     sample_nums = [];
+     epoch_ids = {};
+
+     for i=1:numel(hCfg.rawRecordings),
+         [epochpath, epochfile, epochext] = fileparts(hCfg.rawRecordings{i});
+         epoch_ids{i} = [epochfile epochext];
+         obj = ndiRecording(epoch_ids{i},hCfg);
+         sample_nums(i) = obj.dshape(2);
+         t0_t1s{i} = obj.t0_t1;
+     end;
+
+     % Step 5c: determine the clusters that are "good"
+
+     clusters_to_output = [];
+     for i=1:numel(clusterNotes),
+        if strmpi(c.clusterNotes{i},'noise'), % add other conditions here
+            clusters_to_output(end+1) = i;
+        end;
+     end;
+
+     % Step 5d: write the clusters
+     dependency = struct('name','jrclust_clusters_id','value',d.id());
+         % assume all spikes are eligible to fire for all epochs
+     for i=1:numel(clusters_to_output),
+          element_neuron = ndi.element.timeseries(S,[E.name '_' int2str(clusters_to_output(i))],...
+		E.reference,'spikes',E,0,[],dependency);
+          for j=1:numel(epoch_ids),
+		element_neuron.addepoch(epoch_id,ndi.time.clocktype('dev_local_time'),
+          end;
+     end;
+
+end
+
diff --git a/+jrclust/+utils/+DataHash/DataHash.m b/+jrclust/+utils/+DataHash/DataHash.m
new file mode 100644
index 00000000..ed2d70ad
--- /dev/null
+++ b/+jrclust/+utils/+DataHash/DataHash.m
@@ -0,0 +1,534 @@
+function Hash = DataHash(Data, varargin)
+% DATAHASH - Checksum for Matlab array of any type
+% This function creates a hash value for an input of any type. The type and
+% dimensions of the input are considered as default, such that UINT8([0,0]) and
+% UINT16(0) have different hash values. Nested STRUCTs and CELLs are parsed
+% recursively.
+%
+% Hash = DataHash(Data, Opts...)
+% INPUT:
+%   Data: Array of these built-in types:
+%           (U)INT8/16/32/64, SINGLE, DOUBLE, (real/complex, full/sparse)
+%           CHAR, LOGICAL, CELL (nested), STRUCT (scalar or array, nested),
+%           function_handle, string.
+%   Opts: Char strings to specify the method, the input and theoutput types:
+%         Input types:
+%            'array': The contents, type and size of the input [Data] are
+%                     considered  for the creation of the hash. Nested CELLs
+%                     and STRUCT arrays are parsed recursively. Empty arrays of
+%                     different type reply different hashs.
+%            'file':  [Data] is treated as file name and the hash is calculated
+%                     for the files contents.
+%            'bin':   [Data] is a numerical, LOGICAL or CHAR array. Only the
+%                     binary contents of the array is considered, such that
+%                     e.g. empty arrays of different type reply the same hash.
+%            'ascii': Same as 'bin', but only the 8-bit ASCII part of the 16-bit
+%                     Matlab CHARs is considered.
+%         Output types:
+%            'hex', 'HEX':      Lower/uppercase hexadecimal string.
+%            'double', 'uint8': Numerical vector.
+%            'base64':          Base64.
+%            'short':           Base64 without padding.
+%         Hashing method:
+%            'SHA-1', 'SHA-256', 'SHA-384', 'SHA-512', 'MD2', 'MD5'.
+%            Call DataHash without inputs to get a list of available methods.
+%
+%         Default: 'MD5', 'hex', 'array'
+%
+% OUTPUT:
+%   Hash: String, DOUBLE or UINT8 vector. The length depends on the hashing
+%         method.
+%         If DataHash is called without inputs, a struct is replied:
+%           .HashVersion: Version number of the hashing method of this tool. In
+%              case of bugs or additions, the output can change.
+%           .Date: Date of release of the current HashVersion.
+%           .HashMethod: Cell string of the recognized hash methods.
+%
+% EXAMPLES:
+% % Default: MD5, hex:
+%   DataHash([])                      % 5b302b7b2099a97ba2a276640a192485
+% % MD5, Base64:
+%   DataHash(int32(1:10), 'short', 'MD5')  % +tJN9yeF89h3jOFNN55XLg
+% % SHA-1, Base64:
+%   S.a = uint8([]);
+%   S.b = {{1:10}, struct('q', uint64(415))};
+%   DataHash(S, 'SHA-1', 'HEX')       % 18672BE876463B25214CA9241B3C79CC926F3093
+% % SHA-1 of binary values:
+%   DataHash(1:8, 'SHA-1', 'bin')     % 826cf9d3a5d74bbe415e97d4cecf03f445f69225
+% % SHA-256, consider ASCII part only (Matlab's CHAR has 16 bits!):
+%   DataHash('abc', 'SHA-256', 'ascii')
+%       % ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad
+% % Or equivalently by converting the input to UINT8:
+%   DataHash(uint8('abc'), 'SHA-256', 'bin')
+%
+% NOTES:
+%   Function handles and user-defined objects cannot be converted uniquely:
+%   - The subfunction ConvertFuncHandle uses the built-in function FUNCTIONS,
+%     but the replied struct can depend on the Matlab version.
+%   - It is tried to convert objects to UINT8 streams in the subfunction
+%     ConvertObject. A conversion by STRUCT() might be more appropriate.
+%   Adjust these subfunctions on demand.
+%
+%   MATLAB CHARs have 16 bits! Use Opt.Input='ascii' for comparisons with e.g.
+%   online hash generators.
+%
+%   Matt Raum suggested this for e.g. user-defined objects:
+%     DataHash(getByteStreamFromArray(Data))
+%   This works very well, but unfortunately getByteStreamFromArray is
+%   undocumented, such that it might vanish in the future or reply different
+%   output.
+%
+%   For arrays the calculated hash value might be changed in new versions.
+%   Calling this function without inputs replies the version of the hash.
+%
+%   The older style for input arguments is accepted also: Struct with fields
+%     'Input', 'Method', 'OutFormat'.
+%
+%   The C-Mex function GetMD5 is 2 to 100 times faster, but obtains MD5 only:
+%   http://www.mathworks.com/matlabcentral/fileexchange/25921
+%
+% Tested: Matlab 2009a, 2015b(32/64), 2016b, 2018b, Win7/10
+% Author: Jan Simon, Heidelberg, (C) 2011-2019 matlab.2010(a)n(MINUS)simon.de
+%
+% See also: TYPECAST, CAST.
+%
+% Michael Kleder, "Compute Hash", no structs and cells:
+%   http://www.mathworks.com/matlabcentral/fileexchange/8944
+% Tim, "Serialize/Deserialize", converts structs and cells to a byte stream:
+%   http://www.mathworks.com/matlabcentral/fileexchange/29457
+
+% $JRev: R-R V:043 Sum:VbfXFn6217Hp Date:18-Apr-2019 12:11:42 $
+% $License: BSD (use/copy/change/redistribute on own risk, mention the author) $
+% $UnitTest: uTest_DataHash $
+% $File: Tools\GLFile\DataHash.m $
+% History:
+% 001: 01-May-2011 21:52, First version.
+% 007: 10-Jun-2011 10:38, [Opt.Input], binary data, complex values considered.
+% 011: 26-May-2012 15:57, Fixed: Failed for binary input and empty data.
+% 014: 04-Nov-2012 11:37, Consider Mex-, MDL- and P-files also.
+%      Thanks to David (author 243360), who found this bug.
+%      Jan Achterhold (author 267816) suggested to consider Java objects.
+% 016: 01-Feb-2015 20:53, Java heap space exhausted for large files.
+%      Now files are process in chunks to save memory.
+% 017: 15-Feb-2015 19:40, Collsions: Same hash for different data.
+%      Examples: zeros(1,1) and zeros(1,1,0)
+%                complex(0) and zeros(1,1,0,0)
+%      Now the number of dimensions is included, to avoid this.
+% 022: 30-Mar-2015 00:04, Bugfix: Failed for strings and [] without TYPECASTX.
+%      Ross found these 2 bugs, which occur when TYPECASTX is not installed.
+%      If you need the base64 format padded with '=' characters, adjust
+%      fBase64_enc as you like.
+% 026: 29-Jun-2015 00:13, Changed hash for STRUCTs.
+%      Struct arrays are analysed field by field now, which is much faster.
+% 027: 13-Sep-2015 19:03, 'ascii' input as abbrev. for Input='bin' and UINT8().
+% 028: 15-Oct-2015 23:11, Example values in help section updated to v022.
+% 029: 16-Oct-2015 22:32, Use default options for empty input.
+% 031: 28-Feb-2016 15:10, New hash value to get same reply as GetMD5.
+%      New Matlab version (at least 2015b) use a fast method for TYPECAST, such
+%      that calling James Tursa's TYPECASTX is not needed anymore.
+%      Matlab 6.5 not supported anymore: MException for CATCH.
+% 033: 18-Jun-2016 14:28, BUGFIX: Failed on empty files.
+%      Thanks to Christian (AuthorID 2918599).
+% 035: 19-May-2018 01:11, STRING type considered.
+% 040: 13-Nov-2018 01:20, Fields of Opt not case-sensitive anymore.
+% 041: 09-Feb-2019 18:12, ismethod(class(V),) to support R2018b.
+% 042: 02-Mar-2019 18:39, base64: in Java, short: Base64 with padding.
+%      Unit test. base64->short.
+
+% OPEN BUGS:
+% Nath wrote:
+% function handle refering to struct containing the function will create
+% infinite loop. Is there any workaround ?
+% Example:
+%   d= dynamicprops();
+%   addprop(d,'f');
+%   d.f= @(varargin) struct2cell(d);
+%   DataHash(d.f) % infinite loop
+% This is caught with an error message concerning the recursion limit now.
+
+%#ok<*CHARTEN>
+
+% Reply current version if called without inputs: ------------------------------
+if nargin == 0
+   R = Version_L;
+   
+   if nargout == 0
+      disp(R);
+   else
+      Hash = R;
+   end
+   
+   return;
+end
+
+% Parse inputs: ----------------------------------------------------------------
+[Method, OutFormat, isFile, isBin, Data] = ParseInput(Data, varargin{:});
+
+% Create the engine: -----------------------------------------------------------
+try
+   Engine = java.security.MessageDigest.getInstance(Method);
+   
+catch ME  % Handle errors during initializing the engine:
+   if ~usejava('jvm')
+      Error_L('needJava', 'DataHash needs Java.');
+   end
+   Error_L('BadInput2', 'Invalid hashing algorithm: [%s]. %s', ...
+      Method, ME.message);
+end
+
+% Create the hash value: -------------------------------------------------------
+if isFile
+   [FID, Msg] = fopen(Data, 'r');        % Open the file
+   if FID < 0
+      Error_L('BadFile', ['Cannot open file: %s', char(10), '%s'], Data, Msg);
+   end
+   
+   % Read file in chunks to save memory and Java heap space:
+   Chunk = 1e6;          % Fastest for 1e6 on Win7/64, HDD
+   Count = Chunk;        % Dummy value to satisfy WHILE condition
+   while Count == Chunk
+      [Data, Count] = fread(FID, Chunk, '*uint8');
+      if Count ~= 0      % Avoid error for empty file
+         Engine.update(Data);
+      end
+   end
+   fclose(FID);
+      
+elseif isBin             % Contents of an elementary array, type tested already:
+   if ~isempty(Data)     % Engine.update fails for empty input!
+      if isnumeric(Data)
+         if isreal(Data)
+            Engine.update(typecast(Data(:), 'uint8'));
+         else
+            Engine.update(typecast(real(Data(:)), 'uint8'));
+            Engine.update(typecast(imag(Data(:)), 'uint8'));
+         end
+      elseif islogical(Data)               % TYPECAST cannot handle LOGICAL
+         Engine.update(typecast(uint8(Data(:)), 'uint8'));
+      elseif ischar(Data)                  % TYPECAST cannot handle CHAR
+         Engine.update(typecast(uint16(Data(:)), 'uint8'));
+         % Bugfix: Line removed
+      elseif myIsString(Data)
+         if isscalar(Data)
+            Engine.update(typecast(uint16(Data{1}), 'uint8'));
+         else
+            Error_L('BadBinData', 'Bin type requires scalar string.');
+         end
+      else  % This should have been caught above!
+         Error_L('BadBinData', 'Data type not handled: %s', class(Data));
+      end
+   end
+else                 % Array with type:
+   Engine = CoreHash(Data, Engine);
+end
+
+% Calculate the hash: ----------------------------------------------------------
+Hash = typecast(Engine.digest, 'uint8');
+   
+% Convert hash specific output format: -----------------------------------------
+switch OutFormat
+   case 'hex'
+      Hash = sprintf('%.2x', double(Hash));
+   case 'HEX'
+      Hash = sprintf('%.2X', double(Hash));
+   case 'double'
+      Hash = double(reshape(Hash, 1, []));
+   case 'uint8'
+      Hash = reshape(Hash, 1, []);
+   case 'short'
+      Hash = fBase64_enc(double(Hash), 0);
+   case 'base64'
+      Hash = fBase64_enc(double(Hash), 1);
+      
+   otherwise
+      Error_L('BadOutFormat', ...
+         '[Opt.Format] must be: HEX, hex, uint8, double, base64.');
+end
+
+end
+
+% ******************************************************************************
+function Engine = CoreHash(Data, Engine)
+
+% Consider the type and dimensions of the array to distinguish arrays with the
+% same data, but different shape: [0 x 0] and [0 x 1], [1,2] and [1;2],
+% DOUBLE(0) and SINGLE([0,0]):
+% <  v016: [class, size, data]. BUG! 0 and zeros(1,1,0) had the same hash!
+% >= v016: [class, ndims, size, data]
+Engine.update([uint8(class(Data)), ...
+              typecast(uint64([ndims(Data), size(Data)]), 'uint8')]);
+           
+if issparse(Data)                    % Sparse arrays to struct:
+   [S.Index1, S.Index2, S.Value] = find(Data);
+   Engine                        = CoreHash(S, Engine);
+elseif isstruct(Data)                % Hash for all array elements and fields:
+   F = sort(fieldnames(Data));       % Ignore order of fields
+   for iField = 1:length(F)          % Loop over fields
+      aField = F{iField};
+      Engine.update(uint8(aField));
+      for iS = 1:numel(Data)         % Loop over elements of struct array
+         Engine = CoreHash(Data(iS).(aField), Engine);
+      end
+   end
+elseif iscell(Data)                  % Get hash for all cell elements:
+   for iS = 1:numel(Data)
+      Engine = CoreHash(Data{iS}, Engine);
+   end
+elseif isempty(Data)                 % Nothing to do
+elseif isnumeric(Data)
+   if isreal(Data)
+      Engine.update(typecast(Data(:), 'uint8'));
+   else
+      Engine.update(typecast(real(Data(:)), 'uint8'));
+      Engine.update(typecast(imag(Data(:)), 'uint8'));
+   end
+elseif islogical(Data)               % TYPECAST cannot handle LOGICAL
+   Engine.update(typecast(uint8(Data(:)), 'uint8'));
+elseif ischar(Data)                  % TYPECAST cannot handle CHAR
+   Engine.update(typecast(uint16(Data(:)), 'uint8'));
+elseif myIsString(Data)              % [19-May-2018] String class in >= R2016b
+   classUint8 = uint8([117, 105, 110, 116, 49, 54]);  % 'uint16'
+   for iS = 1:numel(Data)
+      % Emulate without recursion: Engine = CoreHash(uint16(Data{iS}), Engine)
+      aString = uint16(Data{iS});
+      Engine.update([classUint8, ...
+         typecast(uint64([ndims(aString), size(aString)]), 'uint8')]);
+      if ~isempty(aString)
+         Engine.update(typecast(uint16(aString), 'uint8'));
+      end
+   end
+   
+elseif isa(Data, 'function_handle')
+   Engine = CoreHash(ConvertFuncHandle(Data), Engine);
+elseif (isobject(Data) || isjava(Data)) && ismethod(class(Data), 'hashCode')
+   Engine = CoreHash(char(Data.hashCode), Engine);
+else  % Most likely a user-defined object:
+   try
+      BasicData = ConvertObject(Data);
+   catch ME
+      error(['JSimon:', mfilename, ':BadDataType'], ...
+         '%s: Cannot create elementary array for type: %s\n  %s', ...
+         mfilename, class(Data), ME.message);
+   end
+   
+   try
+      Engine = CoreHash(BasicData, Engine);
+   catch ME
+      if strcmpi(ME.identifier, 'MATLAB:recursionLimit')
+         ME = MException(['JSimon:', mfilename, ':RecursiveType'], ...
+            '%s: Cannot create hash for recursive data type: %s', ...
+            mfilename, class(Data));
+      end
+      throw(ME);
+   end
+end
+
+end
+
+% ******************************************************************************
+function [Method, OutFormat, isFile, isBin, Data] = ParseInput(Data, varargin)
+
+% Default options: -------------------------------------------------------------
+Method    = 'MD5';
+OutFormat = 'hex';
+isFile    = false;
+isBin     = false;
+
+% Check number and type of inputs: ---------------------------------------------
+nOpt = nargin - 1;
+Opt  = varargin;
+if nOpt == 1 && isa(Opt{1}, 'struct')   % Old style Options as struct:
+   Opt  = struct2cell(Opt{1});
+   nOpt = numel(Opt);
+end
+
+% Loop over strings in the input: ----------------------------------------------
+for iOpt = 1:nOpt
+   aOpt = Opt{iOpt};
+   if ~ischar(aOpt)
+      Error_L('BadInputType', '[Opt] must be a struct or chars.');
+   end
+   
+   switch lower(aOpt)
+      case 'file'             % Data contains the file name:
+         isFile = true;
+      case {'bin', 'binary'}  % Just the contents of the data:
+         if (isnumeric(Data) || ischar(Data) || islogical(Data) || ...
+               myIsString(Data)) == 0 || issparse(Data)
+            Error_L('BadDataType', ['[Bin] input needs data type: ', ...
+               'numeric, CHAR, LOGICAL, STRING.']);
+         end
+         isBin = true;
+      case 'array'
+         isBin = false;      % Is the default already
+      case {'asc', 'ascii'}  % 8-bit part of MATLAB CHAR or STRING:
+         isBin = true;
+         if ischar(Data)
+            Data  = uint8(Data);
+         elseif myIsString(Data) && numel(Data) == 1
+            Data  = uint8(char(Data));
+         else
+            Error_L('BadDataType', ...
+               'ASCII method: Data must be a CHAR or scalar STRING.');
+         end
+      case 'hex'
+         if aOpt(1) == 'H'
+            OutFormat = 'HEX';
+         else
+            OutFormat = 'hex';
+         end
+      case {'double', 'uint8', 'short', 'base64'}
+         OutFormat = lower(aOpt);
+      otherwise  % Guess that this is the method:
+         Method = upper(aOpt);
+   end
+end
+
+end
+
+% ******************************************************************************
+function FuncKey = ConvertFuncHandle(FuncH)
+%   The subfunction ConvertFuncHandle converts function_handles to a struct
+%   using the Matlab function FUNCTIONS. The output of this function changes
+%   with the Matlab version, such that DataHash(@sin) replies different hashes
+%   under Matlab 6.5 and 2009a.
+%   An alternative is using the function name and name of the file for
+%   function_handles, but this is not unique for nested or anonymous functions.
+%   If the MATLABROOT is removed from the file's path, at least the hash of
+%   Matlab's toolbox functions is (usually!) not influenced by the version.
+%   Finally I'm in doubt if there is a unique method to hash function handles.
+%   Please adjust the subfunction ConvertFuncHandles to your needs.
+
+% The Matlab version influences the conversion by FUNCTIONS:
+% 1. The format of the struct replied FUNCTIONS is not fixed,
+% 2. The full paths of toolbox function e.g. for @mean differ.
+FuncKey = functions(FuncH);
+
+% Include modification file time and file size. Suggested by Aslak Grinsted:
+if ~isempty(FuncKey.file)
+    d = dir(FuncKey.file);
+    if ~isempty(d)
+        FuncKey.filebytes = d.bytes;
+        FuncKey.filedate  = d.datenum;
+    end
+end
+
+% ALTERNATIVE: Use name and path. The <matlabroot> part of the toolbox functions
+% is replaced such that the hash for @mean does not depend on the Matlab
+% version.
+% Drawbacks: Anonymous functions, nested functions...
+% funcStruct = functions(FuncH);
+% funcfile   = strrep(funcStruct.file, matlabroot, '<MATLAB>');
+% FuncKey    = uint8([funcStruct.function, ' ', funcfile]);
+
+% Finally I'm afraid there is no unique method to get a hash for a function
+% handle. Please adjust this conversion to your needs.
+
+end
+
+% ******************************************************************************
+function DataBin = ConvertObject(DataObj)
+% Convert a user-defined object to a binary stream. There cannot be a unique
+% solution, so this part is left for the user...
+
+try    % Perhaps a direct conversion is implemented:
+   DataBin = uint8(DataObj);
+   
+   % Matt Raum had this excellent idea - unfortunately this function is
+   % undocumented and might not be supported in te future:
+   % DataBin = getByteStreamFromArray(DataObj);
+   
+catch  % Or perhaps this is better:
+   WarnS   = warning('off', 'MATLAB:structOnObject');
+   DataBin = struct(DataObj);
+   warning(WarnS);
+end
+
+end
+
+% ******************************************************************************
+function Out = fBase64_enc(In, doPad)
+% Encode numeric vector of UINT8 values to base64 string.
+
+B64 = org.apache.commons.codec.binary.Base64;
+Out = char(B64.encode(In)).';
+if ~doPad
+   Out(Out == '=') = [];
+end
+
+% Matlab method:
+% Pool = [65:90, 97:122, 48:57, 43, 47];  % [0:9, a:z, A:Z, +, /]
+% v8   = [128; 64; 32; 16; 8; 4; 2; 1];
+% v6   = [32, 16, 8, 4, 2, 1];
+%
+% In = reshape(In, 1, []);
+% X  = rem(floor(bsxfun(@rdivide, In, v8)), 2);
+% d6 = rem(numel(X), 6);
+% if d6 ~= 0
+%    X = [X(:); zeros(6 - d6, 1)];
+% end
+% Out = char(Pool(1 + v6 * reshape(X, 6, [])));
+%
+% p = 3 - rem(numel(Out) - 1, 4);
+% if doPad && p ~= 0  % Standard base64 string with trailing padding:
+%    Out = [Out, repmat('=', 1, p)];
+% end
+
+end
+
+% ******************************************************************************
+function T = myIsString(S)
+% isstring was introduced in R2016:
+persistent hasString
+if isempty(hasString)
+   matlabVer = [100, 1] * sscanf(version, '%d.', 2);
+   hasString = (matlabVer >= 901);  % isstring existing since R2016b
+end
+
+T = hasString && isstring(S);  % Short-circuting
+
+end
+
+% ******************************************************************************
+function R = Version_L()
+% The output differs between versions of this function. So give the user a
+% chance to recognize the version:
+% 1: 01-May-2011, Initial version
+% 2: 15-Feb-2015, The number of dimensions is considered in addition.
+%    In version 1 these variables had the same hash:
+%    zeros(1,1) and zeros(1,1,0), complex(0) and zeros(1,1,0,0)
+% 3: 29-Jun-2015, Struct arrays are processed field by field and not element
+%    by element, because this is much faster. In consequence the hash value
+%    differs, if the input contains a struct.
+% 4: 28-Feb-2016 15:20, same output as GetMD5 for MD5 sums. Therefore the
+%    dimensions are casted to UINT64 at first.
+%    19-May-2018 01:13, STRING type considered.
+R.HashVersion = 4;
+R.Date        = [2018, 5, 19];
+
+R.HashMethod  = {};
+try
+   Provider = java.security.Security.getProviders;
+   for iProvider = 1:numel(Provider)
+      S     = char(Provider(iProvider).getServices);
+      Index = strfind(S, 'MessageDigest.');
+      for iDigest = 1:length(Index)
+         Digest       = strtok(S(Index(iDigest):end));
+         Digest       = strrep(Digest, 'MessageDigest.', '');
+         R.HashMethod = cat(2, R.HashMethod, {Digest});
+      end
+   end
+catch ME
+   fprintf(2, '%s\n', ME.message);
+   R.HashMethod = 'error';
+end
+
+end
+
+% ******************************************************************************
+function Error_L(ID, varargin)
+
+error(['JSimon:', mfilename, ':', ID], ['*** %s: ', varargin{1}], ...
+   mfilename, varargin{2:nargin - 1});
+
+end
diff --git a/+jrclust/+utils/+DataHash/license.txt b/+jrclust/+utils/+DataHash/license.txt
new file mode 100644
index 00000000..981079dd
--- /dev/null
+++ b/+jrclust/+utils/+DataHash/license.txt
@@ -0,0 +1,25 @@
+Copyright (c) 2018-2019, Jan Simon
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution
+* Neither the name of  nor the names of its
+  contributors may be used to endorse or promote products derived from this
+  software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/+jrclust/+utils/+DataHash/uTest_DataHash.m b/+jrclust/+utils/+DataHash/uTest_DataHash.m
new file mode 100644
index 00000000..b273bfc7
--- /dev/null
+++ b/+jrclust/+utils/+DataHash/uTest_DataHash.m
@@ -0,0 +1,367 @@
+function uTest_DataHash(doSpeed)
+% Automatic test: DataHash (Mex)
+% This is a routine for automatic testing. It is not needed for processing and
+% can be deleted or moved to a folder, where it does not bother.
+%
+% uTest_DataHash(doSpeed)
+% INPUT:
+%   doSpeed: Optional logical flag to trigger time consuming speed tests.
+%            Default: TRUE. If no speed test is defined, this is ignored.
+% OUTPUT:
+%   On failure the test stops with an error.
+%   The speed is compared to a Java method.
+%
+% Tested: Matlab 2009a, 2015b(32/64), 2016b, 2018b, Win7/10
+% Author: Jan Simon, Heidelberg, (C) 2009-2019 matlab.2010(a)n(MINUS)simon.de
+
+% $JRev: R-e V:004 Sum:cYYIAiiAf7sM Date:19-May-2019 16:58:59 $
+% $License: BSD (use/copy/change/redistribute on own risk, mention the author) $
+% $File: Tools\UnitTests_\uTest_DataHash.m $
+% History:
+% 001: 02-Mar-2019 19:22, First version.
+
+%#ok<*STRQUOT>   % Accept string('s') for R2016b
+%#ok<*STRCLQT>
+ 
+% Initialize: ==================================================================
+% Global Interface: ------------------------------------------------------------
+ErrID = ['JSimon:', mfilename];
+
+MatlabV = [100, 1] * sscanf(version, '%d.', 2);
+
+% Initial values: --------------------------------------------------------------
+if nargin == 0
+   doSpeed = true;
+end
+
+% Program Interface: -----------------------------------------------------------
+% User Interface: --------------------------------------------------------------
+% Do the work: =================================================================
+fprintf('==== Test DataHash:  %s\n', datestr(now, 0));
+fprintf('  Matlab: %s\n',   version);
+
+fprintf('  Java:   %s\n\n', version('-java'));
+
+% Known answer tests - see RFC1321: --------------------------------------------
+disp('== Known answer tests:');
+
+S.a = uint8([]);
+S.b = {{1:10}, struct('q', uint64(415))};
+
+TestData = { ...
+   ... % Desc, Data, Opt, Result:
+   '[]', [], {}, '5b302b7b2099a97ba2a276640a192485'; ...
+   ...
+   'int32(1:10), short, MD5', int32(1:10), {'short', 'MD5'}, ...
+   '+tJN9yeF89h3jOFNN55XLg'; ...
+   ...
+   'int32(1:10), short, MD5, Opt as struct', int32(1:10), ...
+   {struct('Format', 'short', 'Method', 'MD5')}, ...
+   '+tJN9yeF89h3jOFNN55XLg'; ...
+   ...
+   'Struct, HEX, SHA-1', S, ...
+   {'HEX', 'SHA-1'}, '18672BE876463B25214CA9241B3C79CC926F3093'; ...
+   ...
+   'Binary, SHA-1', 1:8, {'SHA-1', 'bin'}, ...
+   '826cf9d3a5d74bbe415e97d4cecf03f445f69225'; ...
+   ...
+   '''abc'', SHA-256, ASCII', 'abc', {'SHA-256', 'ascii'}, ...
+   'ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad'; ...
+   ...
+   'uint8(''abc''), SHA-256, ASCII', uint8('abc'), {'SHA-256', 'bin'}, ...
+   'ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad'; ...
+   ...
+   'message digest, MD5, bin', 'message digest', {'MD5', 'ascii'}, ...
+   'f96b697d7cb7938d525a2f31aaf161d0'; ...
+   ...
+   'char(0:255), MD5, bin', char(0:255), {'MD5', 'ascii'}, ...
+   'e2c865db4162bed963bfaa9ef6ac18f0'; ...
+   ...
+   'char(0:255), MD5, bin', char(0:255), {'MD5', 'Array'}, ...
+   '1e4558b49c05611cdb280a79cb2dbe34'; ...
+   ...
+   '[], SHA-256, base64', 1:33, {'SHA-256', 'base64'}, ...
+   'SuuPZKpce2KetxeIClt1EXz/mCztEuYiawG9vaxKcfc='; ...
+   ...
+   '[], SHA-256, base64', 1:33, {'SHA-512', 'base64'}, ...
+   ['V9Yp/ZBUbfmzQZ7WRzRvqNYLbb6Kzgrc3iaqPH7ta8MS/bMPKc7j', ...
+   '+FRV5Oexu3OCOQ1+2p/E+ZcgKyRHheq0kQ==']; ...
+   ...
+   '[], MD5, short', 1:33, {'MD5', 'short'}, ...
+   'RoNguVVzgq6s7ll9xoCSSg'; ...
+   ...
+   '[], SHA-256, short', 1:33, {'SHA-512', 'short'}, ...
+   ['V9Yp/ZBUbfmzQZ7WRzRvqNYLbb6Kzgrc3iaqPH7ta8MS/bMPKc7j', ...
+   '+FRV5Oexu3OCOQ1+2p/E+ZcgKyRHheq0kQ']; ...
+   ...
+   };
+
+% Create string arrays, if possible:
+if MatlabV >= 901  % R2016b
+   TestData = cat(1, TestData, { ...
+      '"", array', string(''), {'Array'}, ...
+      '061bdd545213c6a236e0f3d655e38ff4'; ...
+      ...
+      '"hello", array', string('hello'), {'Array'}, ...
+      '2614526bcbd4af5a8e7bf79d1d0d92ab'; ...
+      ...
+      '["hello", "world"]', string({'hello', 'world'}), {'Array'}, ...
+      'a1bdbbe9a15c249764847ead9bf47326'; ...
+      ...
+      '["hello"; ""; "world"]', string({'hello'; ''; 'world'}), {'Array'}, ...
+      'a6df2dc811d4e8dab214c01ce0dfc4b9'; ...
+      ...
+      '"", ascii', string(''), {'ascii'}, ...
+      'd41d8cd98f00b204e9800998ecf8427e'; ...
+      ...
+      '"hello", ascii', string('hello'), {'ascii'}, ...
+      '5d41402abc4b2a76b9719d911017c592'; ...
+      ...
+      });
+end
+
+% Run the known answer tests:
+for iTest = 1:size(TestData, 1)
+   Test = TestData(iTest, :);
+   R    = DataHash(Test{2}, Test{3}{:});
+   if isequal(R, Test{4})
+      fprintf('  ok: %s\n', Test{1});
+   else
+      fprintf(2, 'Want: %s\nGot:  %s\n', Test{4}, R);
+      error([ErrID, ':KAT'], 'Failed: %s', Test{1});
+   end
+end
+
+% Create test file:
+TestFile = fullfile(tempdir, [mfilename, '.txt']);
+TestData = {'', 'd41d8cd98f00b204e9800998ecf8427e'; ...
+   ...
+   'abcdefghijklmnopqrstuvwxyz', ...
+   'c3fcd3d76192e4007dfb496cca67e13b'; ...
+   ...
+   'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789', ...
+   'd174ab98d277d9f5a5611c2c9f419d9f'; ...
+   ...
+   ['123456789012345678901234567890123456789012345678901234567890123456', ...
+   '78901234567890'], ...
+   '57edf4a22be3c955ac49da2e2107b67a'; ...
+   ...
+   char(0:255), 'e2c865db4162bed963bfaa9ef6ac18f0'};
+
+try
+   for iTest = 1:size(TestData, 1)
+      Test = TestData(iTest, :);
+      
+      % Create the file:
+      [fid, msg] = fopen(TestFile, 'w');
+      assert(fid ~= -1, msg);
+      fwrite(fid, Test{1});
+      fclose(fid);
+   
+      % Get hash for the file:
+      R    = DataHash(TestFile, 'File');
+      Want = DataHash(Test{1}, 'ascii');
+      if isequal(R, Want, Test{2})
+         fprintf('  ok: empty file\n');
+      else
+         fprintf(2, 'Want: %s\nGot:  %s\n', Want, R);
+         error([ErrID, ':KAT'], 'Failed: File access');
+      end
+   end
+   
+catch ME
+   if exist(TestFile, 'file')
+      delete(TestFile);
+   end
+   rethrow(ME);
+end
+
+delete(TestFile);
+
+% Check different output types: ------------------------------------------------
+N   = 1000;
+B64 = org.apache.commons.codec.binary.Base64;
+   
+disp('== Check output types:');
+for i = 1:N
+   data      = uint8(fix(rand(1, 1 + fix(rand * 100)) * 256));
+   lowHexOut = DataHash(data, 'bin', 'hex');
+   upHexOut  = DataHash(data, 'bin', 'HEX');
+   decOut    = DataHash(data, 'bin', 'Double');
+   base64Out = DataHash(data, 'bin', 'Base64');
+   shortOut  = DataHash(data, 'bin', 'short');
+   uint8Out  = DataHash(data, 'bin', 'Uint8');
+
+   base64pad   = char(B64.encode(decOut)).';
+   base64short = strrep(base64pad, '=', '');
+      
+   if not(strcmpi(lowHexOut, upHexOut) && ...
+         isSame(sscanf(lowHexOut, '%2x'), decOut(:)) && ...
+         isSame(base64Out, base64pad) && ...
+         isSame(shortOut, base64short) && ...
+         isSame(uint8Out, uint8(decOut)))
+      fprintf('\n');
+      error([ErrID, ':Output'], 'Different results for output types.');
+   end
+   
+   % Check binary, e.g. if the data length is a multiple of 2:
+   if rem(length(data), 2) == 0
+      doubleData = double(data);
+      uniData    = char(doubleData(1:2:end) + 256 * doubleData(2:2:end));
+      uniOut     = DataHash(uniData, 'binary', 'double');
+      if not(isequal(uniOut, decOut))
+         error([ErrID, ':Output'], 'Different results for binary mode.');
+      end
+   end
+end
+fprintf(['  ok: %d random tests with hex, HEX, double, uint8, base64 ', ...
+   'output\n'], N);
+
+% Check arrays as inputs: ------------------------------------------------------
+disp('== Test array input:');
+
+% Hash must depend on the type of the array:
+S1 = DataHash([], 'Array');
+if ~isequal(S1, '5b302b7b2099a97ba2a276640a192485')
+   error([ErrID, ':Array'], 'Bad result for array: []');
+end
+
+S1 = DataHash(uint8([]), 'Array');
+if ~isequal(S1, 'cb8a2273d1168a72b70833bb0d79be13')
+   error([ErrID, ':Array'], 'Bad result for array: uint8([])');
+end
+
+S1 = DataHash(int8([]), 'Array');
+if ~isequal(S1, '0160dd4473fe1a952572be239e077ed3')
+   error([ErrID, ':Array'], 'Bad result for array: int8([])');
+end
+
+Data = struct('Field1', 'string', 'Field2', {{'Cell string', '2nd string'}});
+Data.Field3 = Data;
+S1   = DataHash(Data, 'Array');
+if ~isequal(S1, '4fe320b06e3aaaf4ba712980d649e274')
+   error([ErrID, ':Array'], 'Bad result for array: <struct>.');
+end
+
+Data = sparse([1,0,2; 0,3,0; 4, 0,0]);
+S1   = DataHash(Data, 'Array');
+if ~isequal(S1, 'f157bdc9173dff169c782dd639984c82')
+   error([ErrID, ':Array'], 'Bad result for array: <sparse>.');
+end
+fprintf('  ok: Array\n');
+
+% Uninitialized cells contain NULL pointers:
+Data = cell(1, 2);
+S1   = DataHash(Data, 'Array');
+if ~isequal(S1, '161842037bc65b9f3bceffdeb4a8d3bd')
+   error([ErrID, ':Array'], 'Bad result for {NULL, NULL}.');
+end
+fprintf('  ok: Null pointer\n');
+
+% Check string type:
+if MatlabV >= 901  % R2016b
+   Data = string('hello');
+   S1   = DataHash(Data, 'Array');
+   if ~isequal(S1, '2614526bcbd4af5a8e7bf79d1d0d92ab')
+      error([ErrID, ':String'], 'Bad result for string.');
+   end
+   
+   Data = string({'hello', 'world'});
+   S1   = DataHash(Data, 'Array');
+   if ~isequal(S1, 'a1bdbbe9a15c249764847ead9bf47326')
+      error([ErrID, ':String'], 'Bad result for string.');
+   end
+   fprintf('  ok: String class\n');
+end
+
+% Speed test: ------------------------------------------------------------------
+if doSpeed
+   disp('== Test speed:');
+   disp('  * Slower for shorter data due to overhead of calling a function!');
+   disp('  * Process data in memory, not from disk');
+   Delay = 2;
+   
+   % Compare speed with the C-mex GetMD5, if available:
+   getmd5_M = which('GetMD5.m');
+   getmd5_X = which(['GetMD5.', mexext]);
+   if ~isempty(getmd5_M) && ~isempty(getmd5_X)
+      Str       = fileread(getmd5_M);
+      hasGetMD5 = any(strfind(Str, 'Author: Jan Simon'));
+   else
+      hasGetMD5 = false;  % [BUGFIX] 17-May-2019, Thanks zmi zmi
+   end
+   
+   if hasGetMD5
+      fprintf('  * Compare with: %s\n', getmd5_X);
+      fprintf('  * DataHash uses Java for the hashing, GetMD5 fast C code\n\n');
+      fprintf('  Data size:     DataHash:      GetMD5:\n');
+   else
+      fprintf('\n  Data size:     DataHash:\n');
+   end
+   
+   for Len = [10, 100, 1000, 10000, 1e5, 1e6, 1e7, 1e8]
+      [Number, Unit] = UnitPrint(Len, false);
+      fprintf('%12s ', [Number, ' ', Unit]);
+      data = uint8(fix(rand(1, Len) * 256));
+      
+      % Measure time:
+      iLoop = 0;
+      Time  = 0;
+      tic;
+      while Time < Delay || iLoop < 2
+         Hash  = DataHash(data, 'binary', 'uint8');
+         iLoop = iLoop + 1;
+         Time  = toc;
+      end
+      LoopPerSec     = iLoop / Time;
+      [Number, Unit] = UnitPrint(LoopPerSec * Len, true);
+      
+      if hasGetMD5  % Compare with GetMD5, if available:
+         iLoop = 0;
+         Time  = 0;
+         tic;
+         while Time < Delay || iLoop < 2
+            Hash2 = GetMD5(data, 'binary', 'uint8');
+            iLoop = iLoop + 1;
+            Time  = toc;
+         end
+         LoopPerSec2      = iLoop / Time;
+         [Number2, Unit2] = UnitPrint(LoopPerSec2 * Len, true);
+         
+         fprintf('%8s %s/s  %9s %s/s\n', Number, Unit, Number2, Unit2);
+         
+         % Compare the results:
+         if ~isequal(Hash, Hash2)
+            error([ErrID, ':Compare'], 'Result differs from GetMD5.');
+         end
+         
+      else
+         fprintf('%8s %s/s\n', Number, Unit);
+      end
+   end
+end
+
+fprintf('\n== DataHash passed the tests.\n');
+   
+end
+
+% ******************************************************************************
+function E = isSame(A, B)
+E = isequal(A, B) && strcmp(class(A), class(B));
+end
+
+% ******************************************************************************
+function [Number, Unit] = UnitPrint(N, useMB)
+
+if N >= 1e6 || useMB
+   Number = sprintf('%.1f', N / 1e6);
+   Unit   = 'MB';
+elseif N >= 1e3
+   Number = sprintf('%.1f', N / 1000);
+   Unit   = 'kB';
+else
+   Number = sprintf('%g', N);
+   Unit   = ' B';
+end
+
+end
diff --git a/+jrclust/+utils/assign.m b/+jrclust/+utils/assign.m
new file mode 100644
index 00000000..2d17c9a4
--- /dev/null
+++ b/+jrclust/+utils/assign.m
@@ -0,0 +1,35 @@
+function assign (varargin)
+% assign - make a list of assignments (matlab 5 or higher)
+%
+%	ASSIGN('VAR1', VAL1, 'VAR2', VAL2, ...) makes the assignments 
+%	VAR1 = VAL1; VAR2 = VAL2; ... in the caller's workspace.
+%
+%	This is most useful when passing in an option list to a
+%	function.  Thus in the function which starts:
+%		function foo(x,y,varargin)
+%		z = 0;
+%		assign(varargin{:});
+%	the variable z can be given a non-default value by calling the
+%	function like so: foo(x,y,'z',4);
+%
+%       If the input is a single structure, then the structure is converted
+%       to a set of NAME/VALUE pairs and interpreted as 'VAR1', VAL1, etc,
+%       where VAR1 is the first field name of the input, VAL1 is the value of the field name,
+%       etc.
+
+
+if numel(varargin)==1,
+    if isstruct(varargin{1}),
+        varargin = struct2namevaluepair(varargin{1});
+    elseif iscell(varargin{1}),
+        varargin = varargin{1};
+    end;
+end;
+
+vars = {varargin{1:2:end}};
+vals = {varargin{2:2:end}};
+
+% use length(vals) not length(vars) in case of odd number of arguments
+for i = 1:length(vals), % changed 1 to a 2
+  assignin('caller', vars{i}, vals{i});
+end

From ada6b300b02d1ea4e6a54aab4fd2e4d1a6be1a00 Mon Sep 17 00:00:00 2001
From: Vincent <vp35@duke.edu>
Date: Thu, 5 Nov 2020 18:15:47 -0500
Subject: [PATCH 07/22] path to parameter file

---
 +jrclust/+import/kilosort.m | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/+jrclust/+import/kilosort.m b/+jrclust/+import/kilosort.m
index b553ae1f..94b58945 100644
--- a/+jrclust/+import/kilosort.m
+++ b/+jrclust/+import/kilosort.m
@@ -22,7 +22,7 @@
 
 % check for existence of .prm file. if exists use it as a template.
 [a,b,~] = fileparts(params.dat_path);
-prm_path = [a,filesep,b,'.prm'];
+prm_path = fullfile(a,filesep,[b,'.prm']);
 if exist(prm_path,'file')
     cfgData.template_file = prm_path;
 else

From 5e12780112e17934a306f4c689749e940e0bb501 Mon Sep 17 00:00:00 2001
From: "Stephen D. Van Hooser" <vanhoosr@brandeis.edu>
Date: Fri, 6 Nov 2020 06:59:30 -0500
Subject: [PATCH 08/22] export to NDI finished, undergoing testing

---
 +jrclust/+export/ndi.m | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/+jrclust/+export/ndi.m b/+jrclust/+export/ndi.m
index fdb12852..4c05ac27 100644
--- a/+jrclust/+export/ndi.m
+++ b/+jrclust/+export/ndi.m
@@ -161,7 +161,7 @@ function ndi(hCfg, varargin)
      for i=1:numel(hCfg.rawRecordings),
          [epochpath, epochfile, epochext] = fileparts(hCfg.rawRecordings{i});
          epoch_ids{i} = [epochfile epochext];
-         obj = ndiRecording(epoch_ids{i},hCfg);
+         obj = jrclust.detect.ndiRecording(epoch_ids{i},hCfg);
          sample_nums(i) = obj.dshape(2);
          t0_t1s{i} = obj.t0_t1;
      end;
@@ -169,12 +169,14 @@ function ndi(hCfg, varargin)
      % Step 5c: determine the clusters that are "good"
 
      clusters_to_output = [];
-     for i=1:numel(clusterNotes),
-        if strmpi(c.clusterNotes{i},'noise'), % add other conditions here
+     for i=1:numel(c.clusterNotes),
+        if ~strcmpi(c.clusterNotes{i},'noise'), % add other conditions here
             clusters_to_output(end+1) = i;
         end;
      end;
 
+     sample_nums = [1 sample_nums(:)' Inf];
+
      % Step 5d: write the clusters
      dependency = struct('name','jrclust_clusters_id','value',d.id());
          % assume all spikes are eligible to fire for all epochs
@@ -182,7 +184,12 @@ function ndi(hCfg, varargin)
           element_neuron = ndi.element.timeseries(S,[E.name '_' int2str(clusters_to_output(i))],...
 		E.reference,'spikes',E,0,[],dependency);
           for j=1:numel(epoch_ids),
-		element_neuron.addepoch(epoch_id,ndi.time.clocktype('dev_local_time'),
+                local_sample_indexes = find(c.spikesByCluster{clusters_to_output(i)} >= sample_nums(j) & ...
+                        c.spikesByCluster{clusters_to_output(i)} <= sample_nums(j+1));
+                local_sample = c.spikesByCluster{clusters_to_output(i)}(local_sample_indexes) + 1 - sample_nums(j); % convert to local samples
+                spike_times_in_epoch = E.samples2times(epoch_ids{j},local_sample);
+		element_neuron.addepoch(epoch_ids{j},ndi.time.clocktype('dev_local_time'),...
+			t0_t1s{j},spike_times_in_epoch(:),ones(size(spike_times_in_epoch(:))));
           end;
      end;
 

From 96600d99be05efa3368d33bc5c1aa2e05e45f47d Mon Sep 17 00:00:00 2001
From: Steve Van Hooser <vanhoosr@brandeis.edu>
Date: Thu, 12 Nov 2020 21:27:20 -0500
Subject: [PATCH 09/22] ndi updates, also adding neuron_extracellular doc

---
 +jrclust/+export/ndi.m | 30 ++++++++++++++++++++++++++----
 1 file changed, 26 insertions(+), 4 deletions(-)

diff --git a/+jrclust/+export/ndi.m b/+jrclust/+export/ndi.m
index 4c05ac27..5ee5ef43 100644
--- a/+jrclust/+export/ndi.m
+++ b/+jrclust/+export/ndi.m
@@ -180,14 +180,36 @@ function ndi(hCfg, varargin)
      % Step 5d: write the clusters
      dependency = struct('name','jrclust_clusters_id','value',d.id());
          % assume all spikes are eligible to fire for all epochs
+
+     matlab_ver = ver('MATLAB');
+     matlab_version = matlab_ver.Version;
+     app_struct = struct('name', 'JRCLUST', 'version', jrclust.utils.version, 'url', 'https://github.com/JaneliaSciComp/JRCLUST', ...
+         'os', computer, 'os_version', '', 'interpreter', 'MATLAB', 'interpreter_version', matlab_version);
+
      for i=1:numel(clusters_to_output),
           element_neuron = ndi.element.timeseries(S,[E.name '_' int2str(clusters_to_output(i))],...
 		E.reference,'spikes',E,0,[],dependency);
+          neuron_extracellular.number_of_samples_per_channel = size(c.meanWfGlobal,1);
+          neuron_extracellular.number_of_channels = size(c.meanWfGlobal,2);
+          neuron_extracellular.mean_waveform = squeeze(c.meanWfGlobal(:,:,clusters_to_output(i)));
+          neuron_extracellular.waveform_sample_times= [hCfg.evtWindowSamp(1):hCfg.evtWindowSamp(2)] / hCfg.sampleRate;
+          neuron_extracellular.cluster_index = clusters_to_output(i);
+          switch lower(c.clusterNotes{clusters_to_output(i)}),
+               case 'single', value = 1;
+               case 'multi', value = 4;
+               otherwise,
+                   value = -1, % unsure
+          end;
+          neuron_extracellular.quality_number = value;
+          neuron_extracellular.quality_label = c.clusterNotes{clusters_to_output(i)};
+          neuron_doc = ndi.document('neuron/neuron_extracellular.json','app',app_struct,'neuron_extracellular',neuron_extracellular);
+          neuron_doc = neuron_doc.set_dependency_value('element_id',element_neuron.id());
+          S.database_add(neuron_doc);
           for j=1:numel(epoch_ids),
-                local_sample_indexes = find(c.spikesByCluster{clusters_to_output(i)} >= sample_nums(j) & ...
-                        c.spikesByCluster{clusters_to_output(i)} <= sample_nums(j+1));
-                local_sample = c.spikesByCluster{clusters_to_output(i)}(local_sample_indexes) + 1 - sample_nums(j); % convert to local samples
-                spike_times_in_epoch = E.samples2times(epoch_ids{j},local_sample);
+		spike_indexes = c.spikeTimes(c.spikesByCluster{clusters_to_output(i)});
+                local_sample_indexes = find((spike_indexes >= sample_nums(j)) & (spike_indexes <= sample_nums(j+1)));
+                local_sample = spike_indexes(local_sample_indexes) + 1 - sample_nums(j); % convert to local samples
+                spike_times_in_epoch = E.samples2times(epoch_ids{j},double(local_sample));
 		element_neuron.addepoch(epoch_ids{j},ndi.time.clocktype('dev_local_time'),...
 			t0_t1s{j},spike_times_in_epoch(:),ones(size(spike_times_in_epoch(:))));
           end;

From 6402ccbfb567478a1de20760aff6a96057283ef9 Mon Sep 17 00:00:00 2001
From: "Stephen D. Van Hooser" <vanhoosr@brandeis.edu>
Date: Fri, 13 Nov 2020 15:08:43 -0500
Subject: [PATCH 10/22] ndi export added, bootstrap docs added

---
 +jrclust/+export/ndi.m      |  4 +--
 +jrclust/+ndi/discussion.md | 62 -------------------------------------
 docs/usage/tutorial.rst     | 11 +++++++
 3 files changed, 13 insertions(+), 64 deletions(-)
 delete mode 100644 +jrclust/+ndi/discussion.md

diff --git a/+jrclust/+export/ndi.m b/+jrclust/+export/ndi.m
index 5ee5ef43..bc712c67 100644
--- a/+jrclust/+export/ndi.m
+++ b/+jrclust/+export/ndi.m
@@ -187,7 +187,7 @@ function ndi(hCfg, varargin)
          'os', computer, 'os_version', '', 'interpreter', 'MATLAB', 'interpreter_version', matlab_version);
 
      for i=1:numel(clusters_to_output),
-          element_neuron = ndi.element.timeseries(S,[E.name '_' int2str(clusters_to_output(i))],...
+          element_neuron = ndi.neuron(S,[E.name '_' int2str(clusters_to_output(i))],...
 		E.reference,'spikes',E,0,[],dependency);
           neuron_extracellular.number_of_samples_per_channel = size(c.meanWfGlobal,1);
           neuron_extracellular.number_of_channels = size(c.meanWfGlobal,2);
@@ -205,8 +205,8 @@ function ndi(hCfg, varargin)
           neuron_doc = ndi.document('neuron/neuron_extracellular.json','app',app_struct,'neuron_extracellular',neuron_extracellular);
           neuron_doc = neuron_doc.set_dependency_value('element_id',element_neuron.id());
           S.database_add(neuron_doc);
+          spike_indexes = c.spikeTimes(c.spikesByCluster{clusters_to_output(i)});
           for j=1:numel(epoch_ids),
-		spike_indexes = c.spikeTimes(c.spikesByCluster{clusters_to_output(i)});
                 local_sample_indexes = find((spike_indexes >= sample_nums(j)) & (spike_indexes <= sample_nums(j+1)));
                 local_sample = spike_indexes(local_sample_indexes) + 1 - sample_nums(j); % convert to local samples
                 spike_times_in_epoch = E.samples2times(epoch_ids{j},double(local_sample));
diff --git a/+jrclust/+ndi/discussion.md b/+jrclust/+ndi/discussion.md
deleted file mode 100644
index e428daf1..00000000
--- a/+jrclust/+ndi/discussion.md
+++ /dev/null
@@ -1,62 +0,0 @@
-# Discussion of making JRCLUST work with NDI
-
-There are 2 ways to create the user experience. First, one might have the user launch JRCLUST and just
-specify NDI data as the target of analysis. Second, one might have NDI launch JRCLUST.
-
-Let's do the first one at this time.
-
-Actions:
-
-## Need to make an NDI recording object in JRCLUST (subclass of RawRecording)
-
-Describe here
-
-## Need to allow parameters to specify an NDI record:
-
-New interpretations and variables needed for *.prm file:
-
-```
-		rawRecordings = {epoch IDs to include}
-		recordingFormat = 'NDI';
-
-		New variable: NDI (a structure)
-		NDI.experimentpath = path to experiment;
-		NDI.element_name = name of the element
-		NDI.reference = reference number of the element
-
-```
-
-The following parameters need to be filled out:
-
-```
-	nChans = 32; % Number of channels stored in recording file (Distinct from the number of AP sites)
-	sampleRate = 20000; % (formerly sRateHz) Sampling rate (in Hz) of raw recording
-
-	probePad = [12, 12]; % (formerly vrSiteHW) Recording contact pad size (in μm) (Height x width)
-	shankMap = [1; 1; 1; 1; 1; 1; 1; 1; 1; 1; 1; 1; 1; 1; 1; 1; 1; 1; 1; 1; 1; 1; 1; 1; 1; 1; 1; 1; 1; 1; 1; 1]; % (formerly viShank_site) Shank ID of each site
-	siteLoc = [0, 0; 0, 5; 0, 10; 0, 15; 0, 20; 0, 25; 0, 30; 0, 35; 0, 40; 0, 45; 0, 50; 0, 55; 0, 60; 0, 65; 0, 70; 0, 75; 0, 80; 0, 85; 0, 90; 0, 95; 0, 100; 0, 105; 0, 110; 0, 115; 0, 120; 0, 125; 0, 130; 0, 135; 0, 140; 0, 145; 0, 150; 0, 155]; % (formerly mrSiteXY) Site locations (in μm) (x values in the first column, y values in the second column)
-	siteMap = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]; % (formerly viSite2Chan) Map of channel index to site ID (The mapping siteMap(i) = j corresponds to the statement 'site i is stored as channel j in the recording')
-
-```
-
-The following parameters can be "bootstrapped" (JRCLUST term for assigning default parameters) as follows:
-
-		PREPROCESSING parameters can go through as defaults
-
-		SPIKE DETECTION PARAMETERS can go through as defaults
-			(User might want to edit nSiteDir)
-
-		The following can all take their default values:
-			FEATURE EXTRACTION PARAMETERS
-			DISPLAY PARAMETERS
-			AUX CHANNEL PARAMETERS
-			LFP PARAMETERS
-			TRACES PARAMETERS
-			PREVIEW PARAMETERS
-			VALIDATION PARAMETERS
-			TRIAL PARAMETERS
-			CLUSTERING PARAMETERS
-			CURATION PARAMETERS
-
-		
-
diff --git a/docs/usage/tutorial.rst b/docs/usage/tutorial.rst
index 3c55605a..3d258387 100644
--- a/docs/usage/tutorial.rst
+++ b/docs/usage/tutorial.rst
@@ -47,6 +47,14 @@ or
 
    jrc bootstrap /path/to/metafile.meta
 
+or
+
+.. code-block:: matlab
+
+   jrc('bootstrap','ndi',ndi_session_obj,ndi_element_timeseries_obj)
+
+to import from the `Neuroscience Data Interface <https://github.com/NDI-matlab>`__.
+
 See the :ref:`bootstrap documentation <bootstrapping>` for details.
 
 You will be guided through a series of prompts in setting up your config file.
@@ -87,6 +95,9 @@ If you select "Yes", you will be prompted to select exactly one probe file with
    If you do **not** have a probe file in your working directory, the dialog will search in the
    default location, JRCLUST/probes, as shown above.
 
+.. note::
+   At present, reading probe geometry information from NDI is not supported but will be in the future.
+
 Confirmation
 ~~~~~~~~~~~~
 

From dd1f530744ab90cacd62929d6364dd32ac0600f2 Mon Sep 17 00:00:00 2001
From: Alan Liddell <alan@vidriotech.com>
Date: Fri, 26 Feb 2021 12:47:51 -0500
Subject: [PATCH 11/22] update README.md

---
 README.md | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 3274551a..740ee177 100644
--- a/README.md
+++ b/README.md
@@ -1,9 +1,14 @@
 # JRCLUST
 
+## Note
+
+**JRCLUST is no longer being actively maintained.**
+If you still want to use JRCLUST, we recommend using the [latest release](https://github.com/JaneliaSciComp/JRCLUST/releases/tag/v4.1.0) or cloning from `main`, rather than `master`.
+
 JRCLUST is a scalable and customizable package for spike sorting on [high-density silicon probes](https://www.nature.com/articles/nature24636).
 It is written in MATLAB and CUDA.
 
-JRCLUST was originally developed by [James Jun](https://www.simonsfoundation.org/team/james-jun/) and is currently maintained by [Vidrio Technologies](https://vidriotechnologies.com).
+JRCLUST was originally developed by [James Jun](https://www.simonsfoundation.org/team/james-jun/).
 
 ## Installing JRCLUST
 

From a0453fd36104d3a83b265b4148d44996e3753058 Mon Sep 17 00:00:00 2001
From: "Stephen D. Van Hooser" <vanhoosr@brandeis.edu>
Date: Tue, 13 Jul 2021 07:50:12 -0400
Subject: [PATCH 12/22] added unit test for ndi bootstrapping, detection

---
 +jrclust/+detect/ndiRecording.m |  3 +++
 +jrclust/+detect/newRecording.m |  1 -
 +jrclust/+export/ndi.m          | 14 +++++++-----
 +jrclust/+test/ndiTest.m        | 19 ++++++++++++++++
 +jrclust/@Config/Config.m       | 40 ++++++++++++++++++---------------
 +jrclust/@Config/loadParams.m   |  7 +++++-
 +jrclust/@Config/save.m         |  9 +++++++-
 @JRC/bootstrapNDI.m             | 21 ++++++++++++++---
 8 files changed, 85 insertions(+), 29 deletions(-)
 create mode 100644 +jrclust/+test/ndiTest.m

diff --git a/+jrclust/+detect/ndiRecording.m b/+jrclust/+detect/ndiRecording.m
index 1bc12944..57c0921b 100644
--- a/+jrclust/+detect/ndiRecording.m
+++ b/+jrclust/+detect/ndiRecording.m
@@ -16,6 +16,9 @@
             %NDIRECORDING Construct an instance of this class
             % set object data type
             obj = obj@jrclust.interfaces.RawRecording(epochname, hCfg);
+            % file not found error is not an error for ndi
+            obj.errMsg = '';
+            obj.isError = 0;
             obj.S = ndi.session.dir(hCfg.ndiPath);
             E = getelements(obj.S,'element.name',hCfg.ndiElementName,'element.reference',hCfg.ndiElementReference);
             if iscell(E) & numel(E)==1,
diff --git a/+jrclust/+detect/newRecording.m b/+jrclust/+detect/newRecording.m
index d6b7c075..b6df9dcb 100644
--- a/+jrclust/+detect/newRecording.m
+++ b/+jrclust/+detect/newRecording.m
@@ -7,7 +7,6 @@
             hRec = jrclust.detect.ndiRecording(rawPath, hCfg);
         otherwise % SpikeGLX .bin/.dat
             hRec = jrclust.detect.Recording(rawPath, hCfg);
-
     end
 end
 
diff --git a/+jrclust/+export/ndi.m b/+jrclust/+export/ndi.m
index bc712c67..18ada0d3 100644
--- a/+jrclust/+export/ndi.m
+++ b/+jrclust/+export/ndi.m
@@ -194,11 +194,15 @@ function ndi(hCfg, varargin)
           neuron_extracellular.mean_waveform = squeeze(c.meanWfGlobal(:,:,clusters_to_output(i)));
           neuron_extracellular.waveform_sample_times= [hCfg.evtWindowSamp(1):hCfg.evtWindowSamp(2)] / hCfg.sampleRate;
           neuron_extracellular.cluster_index = clusters_to_output(i);
-          switch lower(c.clusterNotes{clusters_to_output(i)}),
-               case 'single', value = 1;
-               case 'multi', value = 4;
-               otherwise,
-                   value = -1, % unsure
+          if ischar(c.clusterNotes{clusters_to_output(i)})
+              switch lower(c.clusterNotes{clusters_to_output(i)}),
+                   case 'single', value = 1;
+                   case 'multi', value = 4;
+                   otherwise,
+                       value = -1, % unsure
+              end;
+          else,
+              value = -1;
           end;
           neuron_extracellular.quality_number = value;
           neuron_extracellular.quality_label = c.clusterNotes{clusters_to_output(i)};
diff --git a/+jrclust/+test/ndiTest.m b/+jrclust/+test/ndiTest.m
new file mode 100644
index 00000000..18f305a0
--- /dev/null
+++ b/+jrclust/+test/ndiTest.m
@@ -0,0 +1,19 @@
+classdef ndiTest < matlab.uitest.TestCase & matlab.mock.TestCase
+	methods(Test)
+		function testNDIBootstrapAndDetect(tc)
+			% use mock session
+			S = ndi.session.mock();
+			P = S.getprobes('type','n-trode');
+			E = S.getelements('element.type','n-trode');
+			E = E{1};
+			jrc('bootstrap','ndi',S,E,'noShow');
+
+			paramdir = dir([S.path() filesep '.JRCLUST' filesep '*_|_*'])
+
+			paramfile = [S.path() filesep '.JRCLUST' filesep paramdir(1).name ...
+				filesep 'jrclust.prm'];
+
+			eval(['jrc detect ' paramfile]);
+		end;
+	end;
+end
diff --git a/+jrclust/@Config/Config.m b/+jrclust/@Config/Config.m
index d6584f35..f83549b3 100644
--- a/+jrclust/@Config/Config.m
+++ b/+jrclust/@Config/Config.m
@@ -301,25 +301,29 @@ function setTemporaryParams(obj, varargin)
                 end
             else
                 % check is a cell array
-                assert(iscell(mr), 'multiRaw must be a cell array');
-
-                % get absolute paths
-                if isprop(obj, 'configFile') && ~isempty(obj.configFile)
-                    basedir = fileparts(obj.configFile);
+                assert(iscell(mr), 'multiRaw must be a cell array'); 
+                % make sure file exist if we are not using ndi; if we are using ndi, then they are epochs which might not have filenames
+                if ~strcmpi(obj.recordingFormat,'ndi'),
+                    % get absolute paths
+                    if isprop(obj, 'configFile') && ~isempty(obj.configFile)
+                        basedir = fileparts(obj.configFile);
+                    else
+                        basedir = pwd();
+                    end
+
+                    mr_ = cellfun(@(fn) jrclust.utils.absPath(fn, basedir), mr, 'UniformOutput', 0);
+
+                    % handle glob expansions
+                    while any(cellfun(@iscell, mr_))
+                        mr_ = [mr_{:}];
+                    end
+
+                    isFound = ~cellfun(@isempty, mr_);
+                    if ~all(isFound)
+                        error('Invalid raw file location in param file.');
+                    end
                 else
-                    basedir = pwd();
-                end
-
-                mr_ = cellfun(@(fn) jrclust.utils.absPath(fn, basedir), mr, 'UniformOutput', 0);
-
-                % handle glob expansions
-                while any(cellfun(@iscell, mr_))
-                    mr_ = [mr_{:}];
-                end
-
-                isFound = ~cellfun(@isempty, mr_);
-                if ~all(isFound)
-                    error('Invalid raw file location in param file.');
+                    mr_ = mr;
                 end
             end
 
diff --git a/+jrclust/@Config/loadParams.m b/+jrclust/@Config/loadParams.m
index 2269f0f3..d6ddb525 100644
--- a/+jrclust/@Config/loadParams.m
+++ b/+jrclust/@Config/loadParams.m
@@ -88,6 +88,11 @@ function loadParams(obj, filename)
 
     % set user-specified params
     uParamNames = fieldnames(userParams);
+    % sort so recordingFormat is first
+    recordingFormatIndex = find(strcmp('recordingFormat',uParamNames));
+    if ~isempty(recordingFormatIndex),
+        uParamNames = uParamNames([recordingFormatIndex; [1:recordingFormatIndex-1]'; [recordingFormatIndex+1:end]']);
+    end;
     for i = 1:numel(uParamNames)
         paramName = uParamNames{i};
 
@@ -122,4 +127,4 @@ function loadParams(obj, filename)
             end
         end
     end
-end
\ No newline at end of file
+end
diff --git a/+jrclust/@Config/save.m b/+jrclust/@Config/save.m
index 45d289ab..e8e8fc2f 100644
--- a/+jrclust/@Config/save.m
+++ b/+jrclust/@Config/save.m
@@ -88,6 +88,13 @@
 
     % write the file
     paramNames = fieldnames(paramsToExport);
+
+    % put recordingFormat first, because some properties depend on this and can't be validated without it
+    recFormatIndex = find(strcmpi('recordingFormat',paramNames));
+    if ~isempty(recFormatIndex),
+        paramNames = paramNames([recFormatIndex; [1:recFormatIndex-1]'; [recFormatIndex+1:end]']);
+    end;
+
     sections = {'usage', 'execution', 'probe', 'recording file', ...
                 'preprocessing', 'spike detection', 'feature extraction', ...
                 'clustering', 'curation', 'display', 'trial', ...
@@ -154,4 +161,4 @@
     end
 
     success = 1;
-end
\ No newline at end of file
+end
diff --git a/@JRC/bootstrapNDI.m b/@JRC/bootstrapNDI.m
index 732dc34b..e6926c90 100644
--- a/@JRC/bootstrapNDI.m
+++ b/@JRC/bootstrapNDI.m
@@ -16,7 +16,7 @@ function bootstrapNDI(obj, varargin)
     % 
     % 
 
-    if numel(varargin)~=2,
+    if (numel(varargin)~=2 & numel(varargin)~=3),
        error(['Usage: jrc(''bootstrap'',''ndi'',ndi_session_dir_obj,ndi_element_timeseries_obj)']);
     end;
 
@@ -51,6 +51,12 @@ function bootstrapNDI(obj, varargin)
        error(ME.message);
     end;
 
+    if numel(varargin)>2,
+        doEdit = varargin{3}; % testing purposes
+    else,
+        doEdit = {'showEdit'};
+    end;
+
     Estring = E.elementstring();
     Estring(find(Estring==' ')) = '_';
     output_dir = [S.path filesep '.JRCLUST' filesep Estring];
@@ -63,6 +69,8 @@ function bootstrapNDI(obj, varargin)
 
     cfgData = struct();
 
+    cfgData.recordingFormat = 'ndi';
+
     [data,t,timeref] = E.readtimeseries(1,0,0); % read 1 sample
     cfgData.tallSkinny = 1; % we will transpose later so this is met
     cfgData.outputDir = output_dir;
@@ -75,11 +83,16 @@ function bootstrapNDI(obj, varargin)
     cfgData.dataTypeRaw = 'int16'; % meaningless
     cfgData.dataTypeExtracted = 'single'; %  we will convert to single resolution
     cfgData.bitScaling = 1;
-    cfgData.recordingFormat = 'ndi';
+    cfgData.recordingFormat = 'ndi'; %this is where it would normally go
     cfgData.ndiPath = S.path;
     cfgData.ndiElementName = E.name;
     cfgData.ndiElementReference = E.reference;
 
+    cfgData.useGPU = 0; % safe setting, user can override
+    cfgData.useParfor = 0; % safe setting, user can override
+    cfgData.maxSecLoad = [ 100 ]; % safe setting, user can override
+    cfgData.nSitesExcl = []; % should be the default but isn't
+
     q = ndi.query('','depends_on',E.id(),'') & ndi.query('','isa','probe file','');
     docs = S.database_search(q);
     if ~isempty(docs),
@@ -103,6 +116,8 @@ function bootstrapNDI(obj, varargin)
     hCfg_.save('', 1);
 
     obj.hCfg = hCfg_;
-    obj.hCfg.edit();
+    if strcmpi(doEdit,'showEdit'),
+        obj.hCfg.edit();
+    end;
 end
 

From b6944b57717b3898e1ece95735e9fbc9853aa542 Mon Sep 17 00:00:00 2001
From: "Stephen D. Van Hooser" <vanhoosr@brandeis.edu>
Date: Sun, 13 Feb 2022 09:30:49 -0500
Subject: [PATCH 13/22] fixed bug in adding multiple directories in ndi

---
 +jrclust/+export/ndi.m | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/+jrclust/+export/ndi.m b/+jrclust/+export/ndi.m
index 18ada0d3..c414a883 100644
--- a/+jrclust/+export/ndi.m
+++ b/+jrclust/+export/ndi.m
@@ -114,13 +114,13 @@ function ndi(hCfg, varargin)
             end;
         else,
             replaceIt = 1; % default will be to replace
-            if interactive,
+            if ~forceReplace &interactive,
                 if interactiveviakey,
                     r = input(['Update clusters to newer version held by JRCLUST? (Y/N): ']);
                     if ~strcmpi(r,'Y') | ~strcmpi(r,'yes'),
                         replaceIt = 0;
                     end;
-                else,
+                elseif ~forceReplace,
                     ButtonName = questdlg(['Update clusters to newer version held by JRCLUST?'],...
                                        'Update?', 'Yes','No/Cancel','Yes');
                     if ~strcmp(ButtonName,'Yes'),
@@ -175,7 +175,7 @@ function ndi(hCfg, varargin)
         end;
      end;
 
-     sample_nums = [1 sample_nums(:)' Inf];
+     sample_nums = [0 cumsum(sample_nums(:)') Inf];
 
      % Step 5d: write the clusters
      dependency = struct('name','jrclust_clusters_id','value',d.id());
@@ -199,11 +199,13 @@ function ndi(hCfg, varargin)
                    case 'single', value = 1;
                    case 'multi', value = 4;
                    otherwise,
+			disp(['Unknown cluster note (will be skipped): ' c.clusterNotes{clusters_to_output(i)} ]);
                        value = -1, % unsure
               end;
           else,
               value = -1;
           end;
+          if value<0, continue; end; % skip the cell if it is not even a multi-unit
           neuron_extracellular.quality_number = value;
           neuron_extracellular.quality_label = c.clusterNotes{clusters_to_output(i)};
           neuron_doc = ndi.document('neuron/neuron_extracellular.json','app',app_struct,'neuron_extracellular',neuron_extracellular);
@@ -211,8 +213,8 @@ function ndi(hCfg, varargin)
           S.database_add(neuron_doc);
           spike_indexes = c.spikeTimes(c.spikesByCluster{clusters_to_output(i)});
           for j=1:numel(epoch_ids),
-                local_sample_indexes = find((spike_indexes >= sample_nums(j)) & (spike_indexes <= sample_nums(j+1)));
-                local_sample = spike_indexes(local_sample_indexes) + 1 - sample_nums(j); % convert to local samples
+                local_sample_indexes = find((spike_indexes > sample_nums(j)) & (spike_indexes <= sample_nums(j+1)));
+                local_sample = spike_indexes(local_sample_indexes) - sample_nums(j); % convert to local samples
                 spike_times_in_epoch = E.samples2times(epoch_ids{j},double(local_sample));
 		element_neuron.addepoch(epoch_ids{j},ndi.time.clocktype('dev_local_time'),...
 			t0_t1s{j},spike_times_in_epoch(:),ones(size(spike_times_in_epoch(:))));

From 67873dd19cad3a325c5b292e6e1473c2712a10a8 Mon Sep 17 00:00:00 2001
From: "Stephen D. Van Hooser" <vanhoosr@brandeis.edu>
Date: Tue, 12 Apr 2022 14:16:23 -0400
Subject: [PATCH 14/22] updates to NDI export

---
 +jrclust/+detect/ndiRecording.m |  2 +-
 +jrclust/+export/ndi.m          | 13 ++++++++++---
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/+jrclust/+detect/ndiRecording.m b/+jrclust/+detect/ndiRecording.m
index 57c0921b..15c4fee3 100644
--- a/+jrclust/+detect/ndiRecording.m
+++ b/+jrclust/+detect/ndiRecording.m
@@ -43,7 +43,7 @@
             match = find(strcmp(epoch_id,{et.epoch_id}));
             if isempty(match),
                  obj.isError = 1;
-                 obj.errMsg = ['No such epoch ' et_here '.'];
+                 obj.errMsg = ['No such epoch ' epoch_id '.'];
                  return;
             end;
             obj.epoch_id = et(match).epoch_id;
diff --git a/+jrclust/+export/ndi.m b/+jrclust/+export/ndi.m
index c414a883..b33cdedc 100644
--- a/+jrclust/+export/ndi.m
+++ b/+jrclust/+export/ndi.m
@@ -51,13 +51,15 @@ function ndi(hCfg, varargin)
     end;
 
        % Step 2: load NDI session and element and check for errors
+
+    disp(['Opening NDI experiment at ' hCfg.ndiPath '...']);
  
     S = ndi.session.dir(hCfg.ndiPath);
     E = getelements(S,'element.name',hCfg.ndiElementName,'element.reference',hCfg.ndiElementReference);
     if iscell(E) & numel(E)==1,
         E = E{1};
     else,
-        error(['No such element ' nCfg.ndiElementName ' with reference ' num2str(hCfg.ndiElementReference) ' found.']);
+        error(['No such element ' hCfg.ndiElementName ' with reference ' num2str(hCfg.ndiElementReference) ' found.']);
     end;
 
        % Step 3: check for existing cluster document; if it exists, ask if we want to delete it
@@ -159,6 +161,7 @@ function ndi(hCfg, varargin)
      epoch_ids = {};
 
      for i=1:numel(hCfg.rawRecordings),
+         disp(['Examining epoch ' hCfg.rawRecordings{i} '...']);
          [epochpath, epochfile, epochext] = fileparts(hCfg.rawRecordings{i});
          epoch_ids{i} = [epochfile epochext];
          obj = jrclust.detect.ndiRecording(epoch_ids{i},hCfg);
@@ -197,10 +200,12 @@ function ndi(hCfg, varargin)
           if ischar(c.clusterNotes{clusters_to_output(i)})
               switch lower(c.clusterNotes{clusters_to_output(i)}),
                    case 'single', value = 1;
+			disp(['Single unit cluster ' int2str(clusters_to_output(i)) ' (will be added): ' c.clusterNotes{clusters_to_output(i)} ]);
                    case 'multi', value = 4;
+			disp(['Multi-unit cluster ' int2str(clusters_to_output(i)) ' (will be added): ' c.clusterNotes{clusters_to_output(i)} ]);
                    otherwise,
-			disp(['Unknown cluster note (will be skipped): ' c.clusterNotes{clusters_to_output(i)} ]);
-                       value = -1, % unsure
+			disp(['Unknown cluster ' int2str(clusters_to_output(i)) ' (will be skipped): ' c.clusterNotes{clusters_to_output(i)} ]);
+                       value = -1; % unsure
               end;
           else,
               value = -1;
@@ -213,11 +218,13 @@ function ndi(hCfg, varargin)
           S.database_add(neuron_doc);
           spike_indexes = c.spikeTimes(c.spikesByCluster{clusters_to_output(i)});
           for j=1:numel(epoch_ids),
+		if ~strcmp(epoch_ids{j},'t00002'),
                 local_sample_indexes = find((spike_indexes > sample_nums(j)) & (spike_indexes <= sample_nums(j+1)));
                 local_sample = spike_indexes(local_sample_indexes) - sample_nums(j); % convert to local samples
                 spike_times_in_epoch = E.samples2times(epoch_ids{j},double(local_sample));
 		element_neuron.addepoch(epoch_ids{j},ndi.time.clocktype('dev_local_time'),...
 			t0_t1s{j},spike_times_in_epoch(:),ones(size(spike_times_in_epoch(:))));
+		end;
           end;
      end;
 

From a991f24c493d9ed55ff5f3d68dce7d17d73345e5 Mon Sep 17 00:00:00 2001
From: "Stephen D. Van Hooser" <vanhoosr@brandeis.edu>
Date: Fri, 15 Apr 2022 11:13:47 -0400
Subject: [PATCH 15/22] added message that cells without notes will be skipped

---
 +jrclust/+export/ndi.m | 1 +
 1 file changed, 1 insertion(+)

diff --git a/+jrclust/+export/ndi.m b/+jrclust/+export/ndi.m
index b33cdedc..fc7620de 100644
--- a/+jrclust/+export/ndi.m
+++ b/+jrclust/+export/ndi.m
@@ -208,6 +208,7 @@ function ndi(hCfg, varargin)
                        value = -1; % unsure
               end;
           else,
+              disp(['Unknown cluster ' int2str(clusters_to_output(i)) ' (will be skipped): (empty note field)']);
               value = -1;
           end;
           if value<0, continue; end; % skip the cell if it is not even a multi-unit

From e6bd01c890458305790cebeeb9b4601c72e714f1 Mon Sep 17 00:00:00 2001
From: "Stephen D. Van Hooser" <vanhoosr@brandeis.edu>
Date: Tue, 20 Sep 2022 17:36:57 -0400
Subject: [PATCH 16/22] fixed ignoring a directory

---
 +jrclust/+export/ndi.m | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/+jrclust/+export/ndi.m b/+jrclust/+export/ndi.m
index fc7620de..4e046083 100644
--- a/+jrclust/+export/ndi.m
+++ b/+jrclust/+export/ndi.m
@@ -219,7 +219,7 @@ function ndi(hCfg, varargin)
           S.database_add(neuron_doc);
           spike_indexes = c.spikeTimes(c.spikesByCluster{clusters_to_output(i)});
           for j=1:numel(epoch_ids),
-		if ~strcmp(epoch_ids{j},'t00002'),
+		if ~strcmp(epoch_ids{j},'pleaseignoreme'),
                 local_sample_indexes = find((spike_indexes > sample_nums(j)) & (spike_indexes <= sample_nums(j+1)));
                 local_sample = spike_indexes(local_sample_indexes) - sample_nums(j); % convert to local samples
                 spike_times_in_epoch = E.samples2times(epoch_ids{j},double(local_sample));

From 05e6bf4950c83f863fff2975b0f3827f697bfbfa Mon Sep 17 00:00:00 2001
From: "Stephen D. Van Hooser" <vanhoosr@brandeis.edu>
Date: Wed, 26 Apr 2023 07:46:56 -0400
Subject: [PATCH 17/22] update for ndi2

---
 +jrclust/+export/ndi.m | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/+jrclust/+export/ndi.m b/+jrclust/+export/ndi.m
index 4e046083..2256dc61 100644
--- a/+jrclust/+export/ndi.m
+++ b/+jrclust/+export/ndi.m
@@ -196,7 +196,8 @@ function ndi(hCfg, varargin)
           neuron_extracellular.number_of_channels = size(c.meanWfGlobal,2);
           neuron_extracellular.mean_waveform = squeeze(c.meanWfGlobal(:,:,clusters_to_output(i)));
           neuron_extracellular.waveform_sample_times= [hCfg.evtWindowSamp(1):hCfg.evtWindowSamp(2)] / hCfg.sampleRate;
-          neuron_extracellular.cluster_index = clusters_to_output(i);
+          neuron_extracellular.waveform_sample_times= neuron_extracellular.waveform_sample_times(:);
+                    neuron_extracellular.cluster_index = clusters_to_output(i);
           if ischar(c.clusterNotes{clusters_to_output(i)})
               switch lower(c.clusterNotes{clusters_to_output(i)}),
                    case 'single', value = 1;

From 18ed885829fb4641f8c81be7eb0c3f2e385eefb5 Mon Sep 17 00:00:00 2001
From: "Stephen D. Van Hooser" <vanhoosr@brandeis.edu>
Date: Mon, 11 Sep 2023 10:51:41 -0400
Subject: [PATCH 18/22] fix to add session id to neurons

---
 +jrclust/+export/ndi.m | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/+jrclust/+export/ndi.m b/+jrclust/+export/ndi.m
index 2256dc61..5b4cd4cd 100644
--- a/+jrclust/+export/ndi.m
+++ b/+jrclust/+export/ndi.m
@@ -142,7 +142,8 @@ function ndi(hCfg, varargin)
      % Step 4: If we are continuing, write the cluster document
 
      d = ndi.document('apps/JRCLUST/jrclust_clusters.json', ...
-         'ndi_document.name', configFileName, ...
+         'base.name', configFileName, ...
+         'base.session_id',S.id(), ...
          'jrclust_clusters.res_mat_MD5_checksum', md5_value);
      d = d.set_dependency_value('element_id',E.id());
      S.database_add(d);
@@ -215,7 +216,8 @@ function ndi(hCfg, varargin)
           if value<0, continue; end; % skip the cell if it is not even a multi-unit
           neuron_extracellular.quality_number = value;
           neuron_extracellular.quality_label = c.clusterNotes{clusters_to_output(i)};
-          neuron_doc = ndi.document('neuron/neuron_extracellular.json','app',app_struct,'neuron_extracellular',neuron_extracellular);
+          neuron_doc = ndi.document('neuron/neuron_extracellular.json','app',app_struct,'neuron_extracellular',neuron_extracellular,...
+		'base.session_id',S.id());
           neuron_doc = neuron_doc.set_dependency_value('element_id',element_neuron.id());
           S.database_add(neuron_doc);
           spike_indexes = c.spikeTimes(c.spikesByCluster{clusters_to_output(i)});

From 41fcc9fac2dce47adce0d55880df7ec97d5605e4 Mon Sep 17 00:00:00 2001
From: "Stephen D. Van Hooser" <vanhoosr@brandeis.edu>
Date: Wed, 1 May 2024 09:33:23 -0400
Subject: [PATCH 19/22] updates for allowing configuration parameters to
 influence data reading

---
 +jrclust/+detect/@DetectController/detectOneRecording.m | 8 ++++----
 +jrclust/+detect/ndiRecording.m                         | 4 ++--
 +jrclust/+interfaces/@RawRecording/RawRecording.m       | 2 +-
 +jrclust/+views/@PreviewController/PreviewController.m  | 2 +-
 +jrclust/+views/@TracesController/TracesController.m    | 4 ++--
 +jrclust/+views/plotAuxCorr.m                           | 2 +-
 @JRC/bootstrapNDI.m                                     | 1 +
 7 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/+jrclust/+detect/@DetectController/detectOneRecording.m b/+jrclust/+detect/@DetectController/detectOneRecording.m
index 3c56ad4a..0ac0c996 100644
--- a/+jrclust/+detect/@DetectController/detectOneRecording.m
+++ b/+jrclust/+detect/@DetectController/detectOneRecording.m
@@ -43,7 +43,7 @@
             end
 
             % load raw samples
-            iSamplesRaw = hRec.readRawROI(obj.hCfg.siteMap, 1+loadOffset:loadOffset+nSamples);
+            iSamplesRaw = hRec.readRawROI(obj.hCfg.siteMap, 1+loadOffset:loadOffset+nSamples,obj.hCfg);
 
             % convert samples to int16
             iSamplesRaw = samplesToInt16(iSamplesRaw, obj.hCfg);
@@ -52,7 +52,7 @@
             if iLoad < nLoads && obj.hCfg.nSamplesPad > 0
                 leftBound = loadOffset + nSamples + 1;
                 rightBound = leftBound + obj.hCfg.nSamplesPad - 1;
-                samplesPost = hRec.readRawROI(obj.hCfg.siteMap, leftBound:rightBound);
+                samplesPost = hRec.readRawROI(obj.hCfg.siteMap, leftBound:rightBound,obj.hCfg);
                 samplesPost = samplesToInt16(samplesPost, obj.hCfg);
             else
                 samplesPost = [];
@@ -135,7 +135,7 @@
         % if in import mode, only process loads where there are imported spikes.    
         if isempty(impTimes) || any(inInterval) 
             % load raw samples
-            iSamplesRaw = hRec.readRawROI(obj.hCfg.siteMap, 1+loadOffset:loadOffset+nSamples);
+            iSamplesRaw = hRec.readRawROI(obj.hCfg.siteMap, 1+loadOffset:loadOffset+nSamples,obj.hCfg);
 
             % convert samples to int16
             iSamplesRaw = samplesToInt16(iSamplesRaw, obj.hCfg);
@@ -144,7 +144,7 @@
             if iLoad < nLoads && obj.hCfg.nSamplesPad > 0
                 leftBound = loadOffset + nSamples + 1;
                 rightBound = leftBound + obj.hCfg.nSamplesPad - 1;
-                samplesPost = hRec.readRawROI(obj.hCfg.siteMap, leftBound:rightBound);
+                samplesPost = hRec.readRawROI(obj.hCfg.siteMap, leftBound:rightBound,obj.hCfg);
                 samplesPost = samplesToInt16(samplesPost, obj.hCfg);
             else
                 samplesPost = [];
diff --git a/+jrclust/+detect/ndiRecording.m b/+jrclust/+detect/ndiRecording.m
index 15c4fee3..6546dbac 100644
--- a/+jrclust/+detect/ndiRecording.m
+++ b/+jrclust/+detect/ndiRecording.m
@@ -73,10 +73,10 @@ function closeRaw(obj)
             obj.rawIsOpen = 0;
         end
 
-        function roi = readRawROI(obj, rows, cols)
+        function roi = readRawROI(obj, rows, cols,hCfg)
             %READRAWROI Get a region of interest by rows/cols from the raw file
             t0t1 = samples2times(obj.E, obj.epoch_id, cols([1 end]));
-            roi = readtimeseries(obj.E, obj.epoch_id, t0t1(1), t0t1(2));
+            roi = hCfg.ndiScale*readtimeseries(obj.E, obj.epoch_id, t0t1(1), t0t1(2));
             roi = roi'; % switch to column-based samples
             roi = single(roi(rows,:)); % if only a subset requested, return only the subset
         end % readRawROI()
diff --git a/+jrclust/+interfaces/@RawRecording/RawRecording.m b/+jrclust/+interfaces/@RawRecording/RawRecording.m
index 7abe9208..b987ed81 100644
--- a/+jrclust/+interfaces/@RawRecording/RawRecording.m
+++ b/+jrclust/+interfaces/@RawRecording/RawRecording.m
@@ -61,7 +61,7 @@
 
     %% ABSTRACT METHODS
     methods (Abstract)
-        roi = readRawROI(obj, rows, cols);
+        roi = readRawROI(obj, rows, cols,configuration);
     end
 
     %% GETTERS/SETTERS
diff --git a/+jrclust/+views/@PreviewController/PreviewController.m b/+jrclust/+views/@PreviewController/PreviewController.m
index 5ac23e2f..336a795e 100644
--- a/+jrclust/+views/@PreviewController/PreviewController.m
+++ b/+jrclust/+views/@PreviewController/PreviewController.m
@@ -255,7 +255,7 @@ function loadPreview(obj)
                 hRec.openRaw();
                 for iLoad = 1:nLoadsFile
                     iBounds = multiBounds{iLoad};
-                    fileTraces_{iLoad} = hRec.readRawROI(obj.hCfg.siteMap, iBounds(1):iBounds(2))';
+                    fileTraces_{iLoad} = hRec.readRawROI(obj.hCfg.siteMap, iBounds(1):iBounds(2),obj.hCfg)';
                 end
                 hRec.closeRaw();
 
diff --git a/+jrclust/+views/@TracesController/TracesController.m b/+jrclust/+views/@TracesController/TracesController.m
index 1b1b1663..b5aba252 100644
--- a/+jrclust/+views/@TracesController/TracesController.m
+++ b/+jrclust/+views/@TracesController/TracesController.m
@@ -90,7 +90,7 @@ function keyPressFigTraces(obj, ~, hEvent)
                     nTimeTraces = obj.hCfg.nSegmentsTraces;
                     multiBounds = jrclust.views.sampleSkip(windowBounds, obj.hFigTraces.figData.nSamplesTotal, nTimeTraces);
 
-                    tracesRaw_ = cellfun(@(lims) obj.hRec.readRawROI(obj.hCfg.siteMap, lims(1):lims(2)), multiBounds, 'UniformOutput', 0);
+                    tracesRaw_ = cellfun(@(lims) obj.hRec.readRawROI(obj.hCfg.siteMap, lims(1):lims(2),obj.hCfg), multiBounds, 'UniformOutput', 0);
                     obj.tracesRaw = jrclust.utils.neCell2mat(tracesRaw_);
                     %if size(tracesRaw_{1},2)~=diff(multiBounds{1})+1, keyboard; end; % sometimes a useful debugging line
 
@@ -289,7 +289,7 @@ function show(obj, recID, showLFP, hClust)
             multiBounds = jrclust.views.sampleSkip(windowBounds, nSamplesTotal, obj.hCfg.nSegmentsTraces);
 
             obj.tracesFull = [];
-            tracesRaw_ = cellfun(@(lims) obj.hRec.readRawROI(obj.hCfg.siteMap, lims(1):lims(2)), multiBounds, 'UniformOutput', 0);
+            tracesRaw_ = cellfun(@(lims) obj.hRec.readRawROI(obj.hCfg.siteMap, lims(1):lims(2),obj.hCfg), multiBounds, 'UniformOutput', 0);
             obj.tracesRaw = jrclust.utils.neCell2mat(tracesRaw_);
 
         %     if obj.hCfg.tallSkinny
diff --git a/+jrclust/+views/plotAuxCorr.m b/+jrclust/+views/plotAuxCorr.m
index 1198f284..cae00027 100644
--- a/+jrclust/+views/plotAuxCorr.m
+++ b/+jrclust/+views/plotAuxCorr.m
@@ -116,7 +116,7 @@
             end
             try
                 hRec = jrclust.detect.newRecording(hCfg.auxFile, hCfg);
-                auxSamples = single(hRec.readRawROI(hCfg.auxChan, 1:hRec.nSamples))*hCfg.bitScaling*hCfg.auxScale;
+                auxSamples = single(hRec.readRawROI(hCfg.auxChan, 1:hRec.nSamples,hCfg))*hCfg.bitScaling*hCfg.auxScale;
                 auxRate = hCfg.getOr('auxRate', hCfg.sampleRate);
             catch % data not from SpikeGLX
                 auxData = load(hCfg.auxFile); %load .mat file containing Aux data 
diff --git a/@JRC/bootstrapNDI.m b/@JRC/bootstrapNDI.m
index e6926c90..1d8e90c8 100644
--- a/@JRC/bootstrapNDI.m
+++ b/@JRC/bootstrapNDI.m
@@ -87,6 +87,7 @@ function bootstrapNDI(obj, varargin)
     cfgData.ndiPath = S.path;
     cfgData.ndiElementName = E.name;
     cfgData.ndiElementReference = E.reference;
+    cfgData.ndiScale = 1;
 
     cfgData.useGPU = 0; % safe setting, user can override
     cfgData.useParfor = 0; % safe setting, user can override

From 12dc18d6d549bd2690354bcec30df800a8ec093f Mon Sep 17 00:00:00 2001
From: Steve Van Hooser <vanhoosr@brandeis.edu>
Date: Tue, 15 Apr 2025 13:53:43 -0400
Subject: [PATCH 20/22] update for new document specification

---
 +jrclust/+export/ndi.m | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/+jrclust/+export/ndi.m b/+jrclust/+export/ndi.m
index 5b4cd4cd..ca3be5d0 100644
--- a/+jrclust/+export/ndi.m
+++ b/+jrclust/+export/ndi.m
@@ -141,7 +141,7 @@ function ndi(hCfg, varargin)
     
      % Step 4: If we are continuing, write the cluster document
 
-     d = ndi.document('apps/JRCLUST/jrclust_clusters.json', ...
+     d = ndi.document('jrclust_clusters', ...
          'base.name', configFileName, ...
          'base.session_id',S.id(), ...
          'jrclust_clusters.res_mat_MD5_checksum', md5_value);

From aae0ccc6950bed40284baf4b1b835fabaaa475b5 Mon Sep 17 00:00:00 2001
From: Steve Van Hooser <vanhoosr@brandeis.edu>
Date: Tue, 15 Apr 2025 13:54:23 -0400
Subject: [PATCH 21/22] update for new document specification

---
 +jrclust/+export/ndi.m | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/+jrclust/+export/ndi.m b/+jrclust/+export/ndi.m
index ca3be5d0..5925ce26 100644
--- a/+jrclust/+export/ndi.m
+++ b/+jrclust/+export/ndi.m
@@ -216,7 +216,7 @@ function ndi(hCfg, varargin)
           if value<0, continue; end; % skip the cell if it is not even a multi-unit
           neuron_extracellular.quality_number = value;
           neuron_extracellular.quality_label = c.clusterNotes{clusters_to_output(i)};
-          neuron_doc = ndi.document('neuron/neuron_extracellular.json','app',app_struct,'neuron_extracellular',neuron_extracellular,...
+          neuron_doc = ndi.document('neuron_extracellular','app',app_struct,'neuron_extracellular',neuron_extracellular,...
 		'base.session_id',S.id());
           neuron_doc = neuron_doc.set_dependency_value('element_id',element_neuron.id());
           S.database_add(neuron_doc);

From b16dc6a12ff7721063cc020a9150f3c76ff96210 Mon Sep 17 00:00:00 2001
From: Steve Van Hooser <vanhoosr@brandeis.edu>
Date: Thu, 10 Jul 2025 15:20:01 -0400
Subject: [PATCH 22/22] update small problem in ndiRecording

---
 +jrclust/+detect/ndiRecording.m | 1 +
 1 file changed, 1 insertion(+)

diff --git a/+jrclust/+detect/ndiRecording.m b/+jrclust/+detect/ndiRecording.m
index 6546dbac..952ee645 100644
--- a/+jrclust/+detect/ndiRecording.m
+++ b/+jrclust/+detect/ndiRecording.m
@@ -62,6 +62,7 @@
             end;
             obj.nsamples = 1+diff(times2samples(obj.E,epoch_id,obj.t0_t1));
             obj.dshape = [hCfg.nChans, obj.nsamples];
+            mksqlite('close');
         end % ndiRecording()
 
         function openRaw(obj)