Merge pull request #1100 from nguyemi5/main

support for dcimg

setup.py

@@ -42,6 +42,10 @@
       "pytest-qt>3.3.0",
 ]
 
+dcimg_deps = [
+    "dcimg @ git+https://github.com/lens-biophotonics/dcimg.git"
+]
+
 # check if pyqt/pyside already installed
 try:
     import PyQt5
@@ -64,7 +68,7 @@
 except:
     pass
 
-all_deps = gui_deps + nwb_deps + test_deps + io_deps
+all_deps = gui_deps + nwb_deps + test_deps + io_deps + dcimg_deps
 
 try:
     import torch
@@ -104,6 +108,7 @@
       ],
       "gui": gui_deps,
       "nwb": nwb_deps,
+      "dcimg": dcimg_deps,
       "io": io_deps,
       "tests": test_deps,
       "all": all_deps,

suite2p/gui/rungui.py

@@ -276,7 +276,7 @@ def create_buttons(self):
         self.inputformat = QComboBox()
         [
             self.inputformat.addItem(f)
-            for f in ["tif", "binary", "bruker", "sbx", "h5", "movie", "nd2", "mesoscan", "raw"]
+            for f in ["tif", "binary", "bruker", "sbx", "h5", "movie", "nd2", "mesoscan", "raw", "dcimg"]
         ]
         self.inputformat.currentTextChanged.connect(self.parse_inputformat)
         self.layout.addWidget(self.inputformat, 2, 0, 1, 1)

suite2p/io/__init__.py

@@ -9,5 +9,6 @@
 from .movie import movie_to_binary
 from .tiff import mesoscan_to_binary, ome_to_binary, tiff_to_binary, generate_tiff_filename, save_tiff
 from .nd2 import nd2_to_binary
+from .dcam import dcimg_to_binary
 from .binary import BinaryFile, BinaryFileCombined
 from .server import send_jobs

suite2p/io/dcam.py

@@ -0,0 +1,110 @@
+"""
+Copyright © 2023 Howard Hughes Medical Institute, Authored by Carsen Stringer and Marius Pachitariu.
+"""
+import os
+import gc
+import math
+import time
+import numpy as np
+from . import utils
+
+try:
+    import dcimg
+    DCIMG = True
+except ImportError:
+    DCIMG = False
+
+
+def dcimg_to_binary(ops):
+    """finds dcimg files and writes them to binaries
+
+    Parameters
+    ----------
+    ops: dictionary
+        "nplanes", "data_path", "save_path", "save_folder", "fast_disk",
+        "nchannels", "keep_movie_raw", "look_one_level_down"
+
+    Returns
+    -------
+        ops : dictionary of first plane
+            ops["reg_file"] or ops["raw_file"] is created binary
+            assigns keys "Ly", "Lx", "tiffreader", "first_tiffs",
+            "nframes", "meanImg", "meanImg_chan2"
+    """
+
+    t0 = time.time()
+    # copy ops to list where each element is ops for each plane
+    ops1 = utils.init_ops(ops)
+
+    # open all binary files for writing
+    # look for dcimg in all requested folders
+    ops1, fs, reg_file, reg_file_chan2 = utils.find_files_open_binaries(ops1, False)
+    ops = ops1[0]
+
+    # loop over all dcimg files
+    iall = 0
+    ik = 0
+
+    for file_name in fs:
+        # open dcimg
+        dcimg_file = dcimg.DCIMGFile(file_name)
+
+        nplanes = 1
+        nchannels = 1
+        nframes = dcimg_file.shape[0]
+
+        iblocks = np.arange(0, nframes, ops1[0]["batch_size"])
+        if iblocks[-1] < nframes:
+            iblocks = np.append(iblocks, nframes)
+
+        if nchannels > 1:
+            nfunc = ops1[0]["functional_chan"] - 1
+        else:
+            nfunc = 0
+
+        # loop over all frames
+        for ichunk, onset in enumerate(iblocks[:-1]):
+            offset = iblocks[ichunk + 1]
+            im_p = dcimg_file[onset:offset, :, :]
+            im2mean = im_p.mean(axis=0).astype(np.float32) / len(iblocks)
+            for ichan in range(nchannels):
+                nframes = im_p.shape[0]
+                im2write = im_p[:]
+                for j in range(0, nplanes):
+                    if iall == 0:
+                        ops1[j]["meanImg"] = np.zeros((im_p.shape[1], im_p.shape[2]),
+                                                      np.float32)
+                        if nchannels > 1:
+                            ops1[j]["meanImg_chan2"] = np.zeros(
+                                (im_p.shape[1], im_p.shape[2]), np.float32)
+                        ops1[j]["nframes"] = 0
+                    if ichan == nfunc:
+                        ops1[j]["meanImg"] += np.squeeze(im2mean)
+                        reg_file[j].write(
+                            bytearray(im2write[:].astype("uint16")))
+                    else:
+                        ops1[j]["meanImg_chan2"] += np.squeeze(im2mean)
+                        reg_file_chan2[j].write(
+                            bytearray(im2write[:].astype("uint16")))
+
+                    ops1[j]["nframes"] += im2write.shape[0]
+            ik += nframes
+            iall += nframes
+
+        dcimg_file.close()
+
+        # write ops files
+    do_registration = ops1[0]["do_registration"]
+    for ops in ops1:
+        ops["Ly"] = dcimg_file.shape[1]
+        ops["Lx"] = dcimg_file.shape[2]
+        if not do_registration:
+            ops["yrange"] = np.array([0, ops["Ly"]])
+            ops["xrange"] = np.array([0, ops["Lx"]])
+        np.save(ops["ops_path"], ops)
+    # close all binary files and write ops files
+    for j in range(0, nplanes):
+        reg_file[j].close()
+        if nchannels > 1:
+            reg_file_chan2[j].close()
+    return ops1[0]

suite2p/io/utils.py

@@ -226,6 +226,26 @@ def get_nd2_list(ops):
         print("** Found %d nd2 files - converting to binary **" % (len(fsall)))
     return fsall, ops
 
+def get_dcimg_list(ops):
+    """ make list of dcimg files to process
+        if ops["look_one_level_down"], then all dcimg"s in all folders + one level down
+    """
+    froot = ops["data_path"]
+    fold_list = ops["data_path"]
+    fsall = []
+    nfs = 0
+    first_tiffs = []
+    for k, fld in enumerate(fold_list):
+        fs, ftiffs = list_files(fld, ops["look_one_level_down"], ["*.dcimg"])
+        fsall.extend(fs)
+        first_tiffs.extend(list(ftiffs))
+    if len(fs) == 0:
+        print("Could not find any dcimg files")
+        raise Exception("no dcimg")
+    else:
+        ops["first_tiffs"] = np.array(first_tiffs).astype("bool")
+        print("** Found %d dcimg files - converting to binary **" % (len(fsall)))
+    return fsall, ops
 
 def find_files_open_binaries(ops1, ish5=False):
     """  finds tiffs or h5 files and opens binaries for writing
@@ -290,6 +310,11 @@ def find_files_open_binaries(ops1, ish5=False):
         fs, ops2 = get_movie_list(ops1[0])
         print("Movie files:")
         print("\n".join(fs))
+    elif input_format == "dcimg":
+        # find dcimgs
+        fs, ops2 = get_dcimg_list(ops1[0])
+        print("DCAM image files:")
+        print("\n".join(fs))
     else:
         # find tiffs
         fs, ops2 = get_tif_list(ops1[0])
@@ -342,7 +367,7 @@ def init_ops(ops):
             ops["save_path"] = os.path.join(ops["save_path0"], "suite2p", "plane%d" % j)
 
         if ("fast_disk" not in ops) or len(ops["fast_disk"]) == 0:
-            ops["fast_disk"] = ops["save_path0"]
+            ops["fast_disk"] = ops["save_path0"].copy()
         fast_disk = os.path.join(ops["fast_disk"], "suite2p", "plane%d" % j)
         ops["ops_path"] = os.path.join(ops["save_path"], "ops.npy")
         ops["reg_file"] = os.path.join(fast_disk, "data.bin")

suite2p/run_s2p.py

@@ -44,6 +44,12 @@
 except ImportError:
     HAS_CV2 = False
 
+try:
+    import dcimg
+    HAS_DCIMG = True
+except ImportError:
+    HAS_DCIMG = False
+
 from functools import partial
 from pathlib import Path
 
@@ -460,6 +466,10 @@ def run_s2p(ops={}, db={}, server={}):
             ops["input_format"] = "nd2"
             if not HAS_ND2:
                 raise ImportError("nd2 not found; pip install nd2")
+        elif ops.get("dcimg"):
+            ops["input_format"] = "dcimg"
+            if not HAS_DCIMG:
+                raise ImportError("dcimg not found; pip install dcimg")
         elif not "input_format" in ops:
             ops["input_format"] = "tif"
         elif ops["input_format"] == "movie":
@@ -484,6 +494,8 @@ def run_s2p(ops={}, db={}, server={}):
                 io.ome_to_binary,
             "movie":
                 io.movie_to_binary,
+            "dcimg":
+                io.dcimg_to_binary,
         }
         if ops["input_format"] in convert_funs:
             ops0 = convert_funs[ops["input_format"]](ops.copy())