Allow to add files to analysis browser which got returned by a python script

.gitattributes

@@ -64,6 +64,7 @@ XOPs-MacOSX-64bit       export-ignore
 
 *.pxp filter=compress
 *.nwb filter=compress
+tools/uv.exe filter=compress
 
 # github specialities, see https://github.com/github/linguist
 Packages/doc/** linguist-documentation

.pre-commit-config.yaml

@@ -36,11 +36,12 @@ repos:
   rev: 0.7.1
   hooks:
   - id: forbid-bidi-controls
-- repo: https://github.com/psf/black-pre-commit-mirror
-  rev: 25.9.0
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.16.2
   hooks:
-    - id: black
-      language_version: python3.13
+    - id: ruff-check
+      args: [ --fix ]
+    - id: ruff-format
 - repo: local
   hooks:
     - id: ipt-format-and-lint

Packages/MIES/MIES_AnalysisBrowser.ipf

@@ -3152,10 +3152,7 @@ Function AB_ButtonProc_AddFolder(STRUCT WMButtonAction &ba) : ButtonControl
 				break
 			endif
 
-			AB_AddElementToSourceList(folder)
-
-			Make/FREE/T wFolder = {folder}
-			AB_AddExperimentEntries(ba.win, wFolder)
+			AB_AddFilesAndFolders(ba.win, {folder})
 			AB_CollapseAll()
 			break
 		default:
@@ -3196,7 +3193,7 @@ Function AB_ButtonProc_AddFiles(STRUCT WMButtonAction &ba) : ButtonControl
 				break
 			endif
 			WAVE/T selFiles = ListToTextWave(fileList, "\r")
-			AB_AddFiles(ba.win, selFiles)
+			AB_AddFilesAndFolders(ba.win, selFiles)
 			AB_CheckFileTypeCheckbox(ba.win, selFiles)
 			AB_CollapseAll()
 			break
@@ -3245,26 +3242,31 @@ static Function AB_CheckFileTypeCheckbox(string win, WAVE/T files)
 	endfor
 End
 
-static Function AB_AddFiles(string win, WAVE/T selFiles)
+/// @brief Add files and folders to the analysis browser
+///
+/// @param win     analysis browser window
+/// @param entries text wave with absolute folder paths containing pxps/nwbs/uxps or absolute paths to files
+///                of that type (backslashes need escaping)
+Function AB_AddFilesAndFolders(string win, WAVE/T entries)
 
 	variable i, index, size
 
-	Duplicate/FREE/T selFiles, newFiles
+	Duplicate/FREE/T entries, newEntries
 
 	WAVE/T folderList = GetAnalysisBrowserGUIFolderList()
-	size = DimSize(selFiles, ROWS)
+	size = DimSize(entries, ROWS)
 	for(i = 0; i < size; i += 1)
-		FindValue/TEXT=selFiles[i]/TXOP=(TXOP_WHOLE_ELEM) folderList
+		FindValue/TEXT=entries[i]/TXOP=(TXOP_WHOLE_ELEM) folderList
 		if(V_Value >= 0)
 			continue
 		endif
-		AB_AddElementToSourceList(selFiles[i])
-		newFiles[index] = selFiles[i]
-		index          += 1
+		AB_AddElementToSourceList(entries[i])
+		newEntries[index] = entries[i]
+		index            += 1
 	endfor
-	Redimension/N=(index) newFiles
+	Redimension/N=(index) newEntries
 
-	AB_AddExperimentEntries(win, newFiles)
+	AB_AddExperimentEntries(win, newEntries)
 End
 
 static Function AB_AddElementToSourceList(string entry)
@@ -3281,6 +3283,10 @@ static Function AB_AddElementToSourceList(string entry)
 	AB_SaveSourceListInSettings()
 End
 
+Function AB_GatherFoldersFromLIMS(WAVE/T cellIDs)
+
+End
+
 static Function AB_SaveSourceListInSettings()
 
 	WAVE/T entryList = GetAnalysisBrowserGUIFolderList()

Packages/MIES/MIES_Python.ipf

@@ -0,0 +1,130 @@
+#pragma TextEncoding     = "UTF-8"
+#pragma rtGlobals        = 3 // Use modern global access method and strict wave access.
+#pragma rtFunctionErrors = 1
+
+#ifdef AUTOMATED_TESTING
+#pragma ModuleName = MIES_PY
+#endif // AUTOMATED_TESTING
+
+/// @brief Return the disc folder where our python scripts are located
+Function/S PY_GetMIESPythonScriptsDiscLocation()
+
+	return GetFolder(FunctionPath("")) + ":Python:"
+End
+
+/// @brief Return the disc folder where the MIES tools are located
+Function/S PY_GetToolsDiscLocation()
+
+	return GetFolder(FunctionPath("")) + "::" + "tools:"
+End
+
+/// @brief Return the disc folder below "Igor Pro XX User Files" for the given package
+Function/S PY_GetPackageFolder(string packageName)
+
+	return GetPythonScriptsFolder() + packageName
+End
+
+/// @brief Return the disc folder to store the virtual environment for the given package
+Function/S PY_GetVirtEnvFolder(string packageName)
+
+	return PY_GetPackageFolder(packageName)
+End
+
+/// @brief Bootstrap a virtual environment for the given package
+///
+/// Use requirements.txt from `tools:<packageName>` to create a virtual
+/// environment at `Igor Pro XX User Files:Python Scripts:<packageName>`.
+///
+/// The venv is completely recreated every time.
+static Function/S PY_CreateVirtEnv(string packageName)
+
+	string venv, cmd, folder, reqFolder, pkgFolder, pyVersion
+	string toolsFolder, uv
+
+	venv        = PY_GetVirtEnvFolder(packageName)
+	pkgFolder   = PY_GetPackageFolder(packageName)
+	toolsFolder = PY_GetToolsDiscLocation()
+	uv          = toolsFolder + "uv.exe"
+	reqFolder   = toolsFolder + packageName + ":"
+	pyVersion   = "3.14"
+
+	sprintf cmd, "%s venv --clear --no-project --no-config --relocatable --managed-python --python %s \"%s\"", HFSPathToWindows(uv), pyVersion, HFSPathToWindows(venv)
+	print cmd
+	ExecuteScriptText/B/Z cmd
+	ASSERT(!V_Flag, "Could not create the python environment")
+
+	sprintf cmd, "%s pip install --no-config --require-hashes --exact --directory \"%s\" --requirements \"%srequirements.txt\"", HFSPathToWindows(uv), HFSPathToWindows(pkgFolder), HFSPathToWindows(reqFolder)
+	print cmd
+	ExecuteScriptText/B/Z cmd
+	ASSERT(!V_Flag, "Could not fill the python environment")
+
+	return venv
+End
+
+/// @brief Activate the virtual environment for the given package
+///
+/// @return 0 on success, 1 if an IP restart is required
+Function PY_ActivateVirtEnv(string packageName)
+
+	string venv, activeVenv
+	variable isPythonRunning
+
+	PythonEnv
+	activeVenv      = StringByKey("NAME", S_PythonEnvInfo, "=", ";")
+	isPythonRunning = V_PythonRunning
+
+	if(!cmpstr(activeVenv, packageName))
+		return 0
+	endif
+
+	venv = PY_CreateVirtEnv(packageName)
+
+	PythonEnv/Z activate=venv
+	ASSERT(!V_flag, "Could not enable the venv for " + packageName)
+
+	if(isPythonRunning)
+		// python was running and we activated a new environment, warn the user
+		print "Igo Pro needs to be restarted as a new Python virtual environment was activated."
+		ControlWindowToFront()
+		return 1
+	endif
+
+	return 0
+End
+
+/// @brief Fetch the disc locations for the given cell names from the
+///        Allen Institute for Brain Science' Lab Information System (LIMS)
+Function/WAVE PY_FetchFilesFromLims(WAVE/T cellnames)
+
+	string list, loc, packageName
+	variable ret
+
+	packageName = "lims-query"
+
+	ret = PY_ActivateVirtEnv(packageName)
+
+	if(ret)
+		return $""
+	endif
+
+	// test without having access to LIMS
+	list = TextWaveToList(cellnames, " ", trailSep = 0)
+	Make/FREE/T/N=0 results
+	loc = PY_GetMIESPythonScriptsDiscLocation() + "limspath_from_cellname.py"
+	PythonFile/Z file=loc, array={"paths", results}, args=list
+	ASSERT(!V_flag, "Error executing LIMS path querying:" + S_PythonError)
+
+	// see DisplayHelpTopic "UNC Paths"
+	results[] = ReplaceRegexInString("^/allen/programs", results[p], "\\\\\\\\allen\\\\programs")
+
+	// forward slashes to backward slashes
+	results[] = ReplaceRegexInString("/", results[p], "\\\\")
+
+	return results
+End
+
+Function dostuff()
+
+	WAVE/T results = PY_FetchFilesFromLims({"fake"})
+	print results
+End

Packages/MIES/MIES_Utilities_System.ipf

@@ -125,6 +125,12 @@ Function/S GetIgorExecutable()
 #endif // IGOR64
 End
 
+/// @brief Return the folder "Python Scripts" usable with Igor Pro 10 or higher
+Function/S GetPythonScriptsFolder()
+
+	return SpecialDirPath("Igor Pro User Files", 0, 0, 0) + "Python Scripts:"
+End
+
 /// @brief Return the number of bits of the architecture
 ///        Igor Pro was built for.
 Function GetArchitectureBits()

Packages/MIES/MIES_WaveDataFolderGetters.ipf

@@ -5797,7 +5797,7 @@ Function/WAVE GetAnalysisBrowserTagsColors()
 	return wv
 End
 
-/// @brief Return the text wave used in the folder listbox of the analysis browser
+/// @brief Return the text wave used in the file/folder listbox of the analysis browser
 Function/WAVE GetAnalysisBrowserGUIFolderList()
 
 	string   name          = "AnaBrowserFolderList"
@@ -5821,7 +5821,7 @@ Function/WAVE GetAnalysisBrowserGUIFolderList()
 	return wv
 End
 
-/// @brief Return the selection wave used in the folder listbox of the analysis browser
+/// @brief Return the selection wave used in the file/folder listbox of the analysis browser
 Function/WAVE GetAnalysisBrowserGUIFolderSelection()
 
 	string   name          = "AnaBrowserFolderSelection"
@@ -5847,7 +5847,7 @@ Function/WAVE GetAnalysisBrowserGUIFolderSelection()
 	return wv
 End
 
-/// @brief Return the color wave used in the folder listbox of the analysis browser
+/// @brief Return the color wave used in the file/folder listbox of the analysis browser
 Function/WAVE GetAnalysisBrowserGUIFolderColors()
 
 	string   name          = "AnaBrowserFolderColors"

Packages/MIES_Include.ipf

@@ -262,6 +262,11 @@ End
 #include "MIES_ProgrammaticGuiControl"
 #include "MIES_Publish"
 #include "MIES_PulseAveraging"
+
+#if IgorVersion() >= 10
+#include "MIES_Python"
+#endif
+
 #include "MIES_RepeatedAcquisition"
 #include "MIES_Replay"
 #include "MIES_SamplingInterval"

Packages/Python/limspath_from_cellname.py

@@ -0,0 +1,62 @@
+import sys
+
+import pg8000
+from pg8000.native import literal
+
+
+def limspath_from_cellname(cellnames: list):
+
+    with (
+        pg8000.connect(
+            user="limsreader",
+            host="limsdb2",
+            database="lims2",
+            password="limsro",
+            port=5432,
+        ) as conn,
+        conn.cursor() as cur,
+    ):
+        paths = []
+
+        for cell in cellnames:
+            cur.execute(
+                f"""SELECT err.storage_directory AS path
+                FROM specimens cell
+                JOIN ephys_roi_results err ON err.id = cell.ephys_roi_result_id
+                WHERE cell.name LIKE {literal(cell)}"""
+            )
+
+            result = cur.fetchone()
+
+            if result != None:
+                paths.append(result[0])
+
+    return paths
+
+
+if __name__ == "__main__":
+    if len(sys.argv) < 2:
+        print("Expected at least one argument: cellnameA cellNameB ...")
+        sys.exit(1)
+
+    cellnames = []
+    paths = []
+
+    # cellnames.append("Pvalb-IRES-Cre;Ai14-791953.03.03.01")
+    cellnames = sys.argv[1:]
+
+    if len(cellnames) == 1 and cellnames[0] == "fake":
+        paths.append(
+            "/allen/programs/celltypes/production/mousecelltypes/prod174/Ephys_Roi_Result_1429085938/"
+        )
+    else:
+        paths = limspath_from_cellname(cellnames)
+
+    for p in paths:
+        if p is None:
+            print("One of the returned paths is None")
+            sys.exit(1)
+
+    print(paths)
+    # @todo Not return zero here due to WM bug #8570
+    # sys.exit(0)

Packages/doc/conf.py

@@ -1,5 +1,5 @@
 # imports
-from subprocess import Popen, PIPE
+from subprocess import PIPE, Popen
 
 
 def setup(app):

Packages/tests/Basic/UTF_Basic_Includes.ipf

@@ -27,6 +27,7 @@
 #include "UTF_oodDAQ"
 #include "UTF_PackageSettings"
 #include "UTF_PGCSetAndActivateControl"
+#include "UTF_Python"
 #include "UTF_StimsetAPI"
 #include "UTF_SweepFormula"
 #include "UTF_SweepFormula_Operations"

Packages/tests/Basic/UTF_Python.ipf

@@ -0,0 +1,11 @@
+#pragma TextEncoding     = "UTF-8"
+#pragma rtGlobals        = 3 // Use modern global access method and strict wave access.
+#pragma rtFunctionErrors = 1
+#pragma ModuleName       = MiesPythonTests
+
+static Function TestLimsPythonScript()
+
+	WAVE results = PY_FetchFilesFromLims({"fake"})
+	Make/T/FREE ref = {"\\\\\\\\allen\\\\programs\\\\celltypes\\\\production\\\\mousecelltypes\\\\prod174\\\\Ephys_Roi_Result_1429085938\\\\"}
+	CHECK_EQUAL_TEXTWAVES(results, ref)
+End

Packages/tests/Basic/UTF_Utils_System.ipf

@@ -171,3 +171,15 @@ static Function TestCacheBackupAndRestoreNoWaves()
 
 	CHECK(IsDataFolderEmpty(dfr))
 End
+
+#if IgorVersion() >= 10
+
+static Function TestGetPythonScriptsFolder()
+
+	string loc = GetPythonScriptsFolder()
+
+	CHECK_PROPER_STR(loc)
+	CHECK(FolderExists(loc))
+End
+
+#endif

Packages/tests/HistoricData/UTF_HistoricDashboard.ipf

@@ -82,7 +82,7 @@ Function TestAnalysisBrowserAddingFiles()
 	PGC_SetAndActivateControl(abWin, "button_AB_Remove")
 	CHECK_EQUAL_VAR(GetNumberFromWaveNote(map, NOTE_INDEX), DimSize(files, ROWS))
 
-	MIES_AB#AB_AddFiles(abWin, {fileToReadd})
+	AB_AddFilesAndFolders(abWin, {fileToReadd})
 	CHECK_EQUAL_VAR(GetNumberFromWaveNote(map, NOTE_INDEX), DimSize(files, ROWS))
 End