Merge pull request #2216 from AllenInstitute/feature/2216-support_mul…

…tiple_devices_for_nwb

Add support for saving each device to its own NWB file

Packages/MIES/MIES_AnalysisBrowser.ipf

@@ -2541,14 +2541,26 @@ static Function AB_RemoveExperimentEntry(string win, string entry)
 	AB_ResetListBoxWaves()
 End
 
+/// @brief returns currently open NWB files for data export. If no file is open returns a zero sized text wave.
+static Function/WAVE AB_GetCurrentlyOpenNWBFiles()
+
+	WAVE/T devicesWithContent = ListToTextWave(GetAllDevicesWithContent(contentType = CONTENT_TYPE_ALL), ";")
+	Duplicate/FREE/T devicesWithContent, activeFiles
+	activeFiles[] = ROStr(GetNWBFilePathExport(devicesWithContent[p]))
+	RemoveTextWaveEntry1D(activeFiles, "", all = 1)
+
+	return activeFiles
+End
+
 static Function AB_AddExperimentEntries(string win, WAVE/T entries)
 
-	string entry, symbPath, fName, nwbFileUsedForExport, panel
+	string entry, symbPath, fName, panel
 	string pxpList, uxpList, nwbList, title
 	variable sTime
 
-	nwbFileUsedForExport = ROStr(GetNWBFilePathExport())
-	panel                = AB_GetPanelName()
+	WAVE/T activeFiles = AB_GetCurrentlyOpenNWBFiles()
+
+	panel = AB_GetPanelName()
 
 	PGC_SetAndActivateControl(win, "button_expand_all", val = 1)
 
@@ -2571,8 +2583,8 @@ static Function AB_AddExperimentEntries(string win, WAVE/T entries)
 		endif
 		for(fName : fileList)
 
-			if(!CmpStr(fName, nwbFileUsedForExport))
-				printf "Ignore %s for adding into the analysis browser\ras we currently export data into it!\r", nwbFileUsedForExport
+			if(!IsNaN(GetRowIndex(activeFiles, str = fName)))
+				printf "Ignore %s for adding into the analysis browser\ras we currently export data into it!\r", fName
 				ControlWindowToFront()
 				continue
 			endif
@@ -3274,6 +3286,16 @@ static Function AB_LoadAllSweepsAndStimsets(string discLocation, string dataFold
 	endfor
 End
 
+static Function/S AB_ReExportGetNewFullFilePath(string fullFilePath, variable numDevices, string device)
+
+	string suffix, name
+
+	suffix = "." + GetFileSuffix(fullFilePath)
+	name   = GetFile(fullFilePath)
+	sprintf name, "%s%s-converted%s", RemoveEnding(name, suffix), SelectString(numDevices > 1, "", "-" + device), suffix
+	return GetFolder(fullFilePath) + name
+End
+
 static Function AB_ReExport(variable index, variable overwrite)
 
 	string fileName, discLocation, dataFolder, experiment, path, name, suffix
@@ -3381,12 +3403,9 @@ static Function AB_ReExport(variable index, variable overwrite)
 		endif
 
 		// Now export all that data into NWBv2
-		suffix = "." + GetFileSuffix(discLocation)
-		name   = GetFile(discLocation)
-		sprintf name, "%s%s-converted%s", RemoveEnding(name, suffix), SelectString(numDevices > 1, "", "-" + device), suffix
-		path = GetFolder(discLocation) + name
+		path = AB_ReExportGetNewFullFilePath(discLocation, numDevices, device)
 
-		ret = NWB_ExportAllData(NWB_VERSION_LATEST, overrideFilePath = path, writeStoredTestPulses = 1, overwrite = overwrite)
+		ret = NWB_ExportAllData(NWB_VERSION_LATEST, overrideFullFilePath = path, writeStoredTestPulses = 1, overwrite = overwrite)
 
 		if(!ret)
 			printf "####\r"
@@ -3422,6 +3441,15 @@ static Function BeforeFileOpenHook(refNum, file, pathName, type, creator, kind)
 	PGC_SetAndActivateControl("AnalysisBrowser", "button_expand_all", val = 1)
 
 	entry = basefolder + file
+
+	WAVE/T activeFiles = AB_GetCurrentlyOpenNWBFiles()
+	if(!IsNaN(GetRowIndex(activeFiles, str = entry)))
+		printf "Can not add dropped file because it is currently open for data export: %s\rTo close the file unlock the device in the respective acquisition panel.", entry
+		ControlWindowToFront()
+		LOG_AddEntry(PACKAGE_MIES, "end")
+		return 1
+	endif
+
 	if(AB_AddFile(entry, entry))
 		// already loaded or error
 		LOG_AddEntry(PACKAGE_MIES, "end")

Packages/MIES/MIES_AnalysisBrowser_SweepBrowser.ipf

@@ -481,6 +481,49 @@ Function/WAVE SB_GetPlainSweepList(win)
 	return sweeps
 End
 
+/// @brief Gets sweep numbers for a given device
+Function/WAVE SB_GetSweepsFromDevice(string win, string device)
+
+	variable numRows, dimDevice, dimSweeps
+
+	WAVE/T map = SB_GetSweepBrowserMapFromGraph(win)
+	numRows = GetNumberFromWaveNote(map, NOTE_INDEX)
+	if(!numRows)
+		return $""
+	endif
+
+	dimDevice = FindDimlabel(map, COLS, "Device")
+	WAVE/Z devIndizes = FindIndizes(map, col = dimDevice, endRow = numRows - 1, str = device)
+	if(!WaveExists(devIndizes))
+		return $""
+	endif
+
+	dimSweeps = FindDimlabel(map, COLS, "Sweep")
+	Duplicate/FREE devIndizes, devSweeps
+	MultiThread devSweeps[] = str2num(map[devIndizes[p]][dimSweeps])
+
+	return devSweeps
+End
+
+/// @brief Gets the unique devices for which data was loaded into the sweepbrowser
+Function/WAVE SB_GetDeviceList(string win)
+
+	variable dimDevice, numRows
+
+	WAVE/T map = SB_GetSweepBrowserMapFromGraph(win)
+	numRows = GetNumberFromWaveNote(map, NOTE_INDEX)
+	if(!numRows)
+		return $""
+	endif
+
+	dimDevice = FindDimlabel(map, COLS, "Device")
+	Duplicate/FREE/RMD=[0, numRows - 1][dimDevice] map, devices
+
+	WAVE/T uniqueEntries = GetUniqueEntries(devices)
+
+	return uniqueEntries
+End
+
 /// @brief Generic getter for the labnotebook waves
 ///
 /// LBT_LABNOTEBOOK:

Packages/MIES/MIES_DAEphys.ipf

@@ -2394,7 +2394,7 @@ Function DAP_CheckSettings(device, mode)
 	endif
 
 	if(DAG_GetNumericalValue(device, "Check_Settings_NwbExport"))
-		NWB_PrepareExport(str2num(DAG_GetTextualValue(device, "Popup_Settings_NwbVersion")))
+		NWB_PrepareExport(str2num(DAG_GetTextualValue(device, "Popup_Settings_NwbVersion")), device)
 	endif
 
 	return 0
@@ -4742,7 +4742,7 @@ static Function DAP_UnlockDevice(device)
 
 	wlName = GetWorkLoadName(WORKLOADCLASS_TP, device)
 	ASSERT(!ASYNC_WaitForWLCToFinishAndRemove(wlName, DAP_WAITFORTPANALYSIS_TIMEOUT), "TP analysis did not finish within timeout")
-	NWB_ASYNC_FinishWriting(device)
+	NWB_CloseNWBFile(device)
 
 	PGC_SetAndActivateControl(device, "check_Settings_TPAfterDAQ", val = state)
 
@@ -4807,7 +4807,6 @@ static Function DAP_UnlockDevice(device)
 
 	lockedDevices = GetListOfLockedDevices()
 	if(IsEmpty(lockedDevices))
-		CloseNWBFile()
 
 		WAVE ActiveDevicesTPMD = GetActiveDevicesTPMD()
 		ActiveDevicesTPMD = NaN

Packages/MIES/MIES_GlobalStringAndVariableAccess.ipf

@@ -526,22 +526,22 @@ Function/S GetSessionStartTime()
 End
 
 /// @brief Return the HDF5 file identifier for the NWB export
-Function/S GetNWBFileIDExport()
+Function/S GetNWBFileIDExport(string device)
 
-	return GetNVARAsString(GetNWBFolder(), "fileIdExport", initialValue = NaN)
+	return GetNVARAsString(GetDevicePath(device), "NWBfileIdExport", initialValue = NaN)
 End
 
 /// @brief Return the absolute path to the file for NWB export
-Function/S GetNWBFilePathExport()
+Function/S GetNWBFilePathExport(string device)
 
-	return GetSVARAsString(GetNWBFolder(), "filePathExport")
+	return GetSVARAsString(GetDevicePath(device), "NWBfilePathExport")
 End
 
 /// @brief Return the experiment session start time in NWB-speech as
 ///        read back from the NWB file.
-Function/S GetSessionStartTimeReadBack()
+Function/S GetSessionStartTimeReadBack(string device)
 
-	return GetNVARAsString(GetNWBFolder(), "sessionStartTimeReadBack", initialValue = NaN)
+	return GetNVARAsString(GetDevicePath(device), "sessionStartTimeReadBack", initialValue = NaN)
 End
 
 /// @brief Return the thread group ID for the FIFO monitor/resetting daemon

Packages/MIES/MIES_IVSCC.ipf

@@ -238,11 +238,18 @@ Function IVS_ExportAllData(filePath)
 
 	printf "Saving experiment data in NWB format to %s\r", filePath
 
-	return NWB_ExportAllData(IVS_DEFAULT_NWBVERSION, overrideFilePath = filePath, overwrite = 1)
+	return NWB_ExportAllData(IVS_DEFAULT_NWBVERSION, overrideFullFilePath = filePath, overwrite = 1)
 End
 
 Function/S IVS_ReturnNWBFileLocation()
-	SVAR path = $GetNWBFilePathExport()
+
+	string device, devicesWithContent
+
+	devicesWithContent = GetAllDevicesWithContent(contentType = CONTENT_TYPE_ALL)
+	ASSERT(ItemsInList(devicesWithContent) == 1, "Only supported for single device operation.")
+	device = StringFromList(0, devicesWithContent)
+	SVAR path = $GetNWBFilePathExport(device)
+
 	return path
 End
 

Packages/MIES/MIES_IgorHooks.ipf

@@ -116,6 +116,8 @@ static Function BeforeExperimentSaveHook(rN, fileName, path, type, creator, kind
 	variable rN, kind
 	string fileName, path, type, creator
 
+	string device
+
 	// don't try cleaning up if the user never used MIES
 	if(!DataFolderExists(GetMiesPathAsString()))
 		return NaN
@@ -128,11 +130,13 @@ static Function BeforeExperimentSaveHook(rN, fileName, path, type, creator, kind
 
 	IH_KillTemporaries()
 
-	NVAR fileIDExport = $GetNWBFileIDExport()
-
-	if(H5_IsFileOpen(fileIDExport))
-		NWB_Flush(fileIDExport)
-	endif
+	WAVE/T devicesWithContent = ListToTextWave(GetAllDevicesWithContent(contentType = CONTENT_TYPE_ALL), ";")
+	for(device : devicesWithContent)
+		NVAR fileIDExport = $GetNWBFileIDExport(device)
+		if(H5_IsFileOpen(fileIDExport))
+			NWB_Flush(fileIDExport)
+		endif
+	endfor
 
 	UpdateXOPLoggingTemplate()
 

Packages/MIES/MIES_MiesUtilities_Device.ipf

@@ -227,28 +227,6 @@ Function/S GetListOfLockedITC1600Devices()
 	return ListMatch(GetListOfLockedDevices(), "ITC1600*")
 End
 
-/// @brief Close a possibly open export-into-NWB file
-///
-/// We have only one NWB file open for all running devices
-Function CloseNWBFile()
-	string lockedDevices
-
-	NVAR fileID = $GetNWBFileIDExport()
-
-	if(IsFinite(fileID))
-		lockedDevices = GetListOfLockedDevices()
-		CallFunctionForEachListItem(NWB_ASYNC_FinishWriting, lockedDevices)
-
-		HDF5CloseFile/Z fileID
-		DEBUGPRINT("Trying to close the NWB file using HDF5CloseFile returned: ", var = V_flag)
-		if(!V_flag) // success
-			fileID = NaN
-			SVAR filePath = $GetNWBFilePathExport()
-			filepath = ""
-		endif
-	endif
-End
-
 /// @brief Check that the device is of type ITC1600
 Function IsITC1600(device)
 	string device

Packages/MIES/MIES_MiesUtilities_Recreation.ipf

@@ -28,7 +28,7 @@ Function RecreateMissingSweepAndConfigWaves(string device, DFREF deviceDataDFR)
 
 	printf "Trying to resurrect missing sweeps from device %s\r", device
 
-	variable i, numEntries, sweepNo, samplingInterval
+	variable i, numEntries, sweepNo, samplingInterval, configVersion
 	string path
 
 	WAVE numericalKeys = GetLBTextualKeys(device)
@@ -99,17 +99,19 @@ Function RecreateMissingSweepAndConfigWaves(string device, DFREF deviceDataDFR)
 				continue
 			endif
 		else
-			WAVE/Z sweepWave
+			WAVE/Z/SDFR=deviceDataDFR sweepWave = $GetSweepWaveName(sweepNo)
 		endif
 
 		if(missingConfig[i])
 			WAVE configWave = RecreateConfigWaveFromLBN(device, numericalValues, textualValues, sweepNo)
 		else
-			WAVE/Z configWave
+			WAVE/Z/SDFR=deviceDataDFR configWave = $GetConfigWaveName(sweepNo)
 		endif
 
 		if(WaveExists(sweepWave) && WaveExists(configWave))
-			ASSERT(IsValidSweepAndConfig(sweepWave, configWave), "Recreation created incompatible sweep and config waves")
+			configVersion = GetWaveVersion(configWave)
+			configVersion = IsNaN(configVersion) ? 0 : configVersion
+			ASSERT(IsValidSweepAndConfig(sweepWave, configWave, configVersion = configVersion), "Recreation created incompatible sweep and config waves")
 		endif
 
 		if(WaveExists(configWave))

Packages/MIES/MIES_MiesUtilities_System.ipf

@@ -148,7 +148,7 @@ Function SaveExperimentSpecial(mode)
 		KillWindow/Z HistoryCarbonCopy
 		CreateHistoryNotebook()
 
-		CloseNWBFile()
+		NWB_CloseAllNWBFiles()
 
 		NVAR sesssionStartTime = $GetSessionStartTime()
 		sesssionStartTime = DateTimeInUTC()