From 568342c4a44236aaaacfa434aedbe6e26ea6b9c1 Mon Sep 17 00:00:00 2001 From: Tim Parker Date: Sun, 10 Apr 2022 13:43:47 -0500 Subject: [PATCH 1/2] Add dry-run support for quality control and output directory for targetted batch QC --- rawtools/cli.py | 37 +++++++++++------ rawtools/log.py | 83 ++++++++++++++++++++------------------ rawtools/qualitycontrol.py | 16 +++++--- 3 files changed, 79 insertions(+), 57 deletions(-) diff --git a/rawtools/cli.py b/rawtools/cli.py index 5b265bb..1b3f21a 100644 --- a/rawtools/cli.py +++ b/rawtools/cli.py @@ -86,18 +86,31 @@ def raw_nsihdr(): def raw_qc(): """Quality control tools""" - description="Check the quality of a .RAW volume by extracting a slice or generating a projection. Requires a .RAW and .DAT for each volume." - - parser = argparse.ArgumentParser(description=description, formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument("-V", "--version", action="version", version=f'%(prog)s {__version__}') - parser.add_argument("-v", "--verbose", action="store_true", help="Increase output verbosity") - parser.add_argument("-f", "--force", action="store_true", default=False, help="Force file creation. Overwrite any existing files.") - parser.add_argument("--si", action="store_true", default=False, help="Print human readable sizes (e.g., 1 K, 234 M, 2 G)") - parser.add_argument("-p", "--projection", action="store", nargs='+', help="Generate projection using maximum values for each slice. Available options: [ 'top', 'side' ].") - parser.add_argument("--scale", dest="step", const=100, action="store", nargs='?', default=argparse.SUPPRESS, type=int, help="Add scale on left side of a side projection. Step is the number of slices between each label. (default: 100)") - parser.add_argument("-s", "--slice", dest='index', const=True, nargs='?', type=int, default=argparse.SUPPRESS, help="Extract a slice from volume's side view. (default: floor(x/2))") - parser.add_argument("--font-size", dest="font_size", action="store", type=int, default=24, help="Font size of labels of scale.") - parser.add_argument("path", metavar='PATH', type=str, nargs='+', help='Filepath to a .RAW or path to a directory that contains .RAW files.') + description = "Check the quality of a .RAW volume by extracting a slice or generating a projection. Requires a .RAW and .DAT for each volume." + + parser = argparse.ArgumentParser( + description=description, formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument("-V", "--version", action="version", + version=f'%(prog)s {__version__}') + parser.add_argument("-v", "--verbose", action="store_true", + help="Increase output verbosity") + parser.add_argument("-f", "--force", action="store_true", default=False, + help="Force file creation. Overwrite any existing files.") + parser.add_argument("--si", action="store_true", default=False, + help="Print human readable sizes (e.g., 1 K, 234 M, 2 G)") + parser.add_argument("-n", '--dry-run', dest='dryrun', action="store_true", + help="Perform a trial run. Do not create image files, but logs will be updated.") + parser.add_argument("-p", "--projection", action="store", nargs='+', + help="Generate projection using maximum values for each slice. Available options: [ 'top', 'side' ].") + parser.add_argument("--scale", dest="step", const=100, action="store", nargs='?', default=argparse.SUPPRESS, type=int, + help="Add scale on left side of a side projection. Step is the number of slices between each label. (default: 100)") + parser.add_argument("-s", "--slice", dest='index', const=True, nargs='?', type=int, + default=argparse.SUPPRESS, help="Extract a slice from volume's side view. (default: floor(x/2))") + parser.add_argument("--font-size", dest="font_size", action="store", + type=int, default=24, help="Font size of labels of scale.") + parser.add_argument("-o", "--outdir", action="store", help="Output folder to store all generated files. If not provided, each file will be placed in the folder of the source data.") + parser.add_argument("path", metavar='PATH', type=str, nargs='+', + help='Filepath to a .RAW or path to a directory that contains .RAW files.') args = parser.parse_args() args.module_name = 'qc' diff --git a/rawtools/log.py b/rawtools/log.py index ba1991a..707d778 100644 --- a/rawtools/log.py +++ b/rawtools/log.py @@ -9,43 +9,46 @@ def configure(args): - """Set up log files and associated handlers""" - # Configure logging, stderr and file logs - logging_level = logging.INFO - if args.verbose: - logging_level = logging.DEBUG - - logFormatter = logging.Formatter("%(asctime)s - [%(levelname)-4.8s] - %(filename)s %(lineno)d - %(message)s") - rootLogger = logging.getLogger() - rootLogger.setLevel(logging.DEBUG) - - # Set project-level logging - if args.module_name is not None: - logfile_basename = f"{dt.today().strftime('%Y-%m-%d_%H-%M-%S')}_{args.module_name}.log" - rpath = os.path.realpath(args.path[0]) - if os.path.isdir(rpath): - dname = rpath - else: - dname = os.path.dirname(rpath) - lfp = os.path.join(dname, logfile_basename) # base log file path - fileHandler = logging.FileHandler(lfp) - fileHandler.setFormatter(logFormatter) - fileHandler.setLevel(logging.DEBUG) # always show debug statements in log file - rootLogger.addHandler(fileHandler) - - sdfp = os.path.join('/', 'var', 'log', 'rawtools', args.module_name) # system directory file path - if not os.path.exists(sdfp): - os.makedirs(sdfp) - slfp = os.path.join(sdfp, logfile_basename) # system log file path - syslogFileHandler = logging.FileHandler(slfp) - syslogFileHandler.setFormatter(logFormatter) - syslogFileHandler.setLevel(logging.DEBUG) # always show debug statements in log file - rootLogger.addHandler(syslogFileHandler) - - consoleHandler = logging.StreamHandler() - consoleHandler.setFormatter(logFormatter) - consoleHandler.setLevel(logging_level) - rootLogger.addHandler(consoleHandler) - - logging.debug(f'Running {args.module_name} {__version__}') - logging.debug(f"Command: {args}") + """Set up log files and associated handlers""" + # Configure logging, stderr and file logs + logging_level = logging.INFO + if args.verbose: + logging_level = logging.DEBUG + + logFormatter = logging.Formatter("%(asctime)s - [%(levelname)-4.8s] - %(filename)s %(lineno)d - %(message)s") + rootLogger = logging.getLogger() + rootLogger.setLevel(logging.DEBUG) + + # Set project-level logging + if args.module_name is not None: + logfile_basename = f"{dt.today().strftime('%Y-%m-%d_%H-%M-%S')}_{args.module_name}.log" + if args.outdir is not None: + rpath = os.path.realpath(args.outdir) + else: + rpath = os.path.realpath(args.path[0]) + if os.path.isdir(rpath): + dname = rpath + else: + dname = os.path.dirname(rpath) + lfp = os.path.join(dname, logfile_basename) # base log file path + fileHandler = logging.FileHandler(lfp) + fileHandler.setFormatter(logFormatter) + fileHandler.setLevel(logging.DEBUG) # always show debug statements in log file + rootLogger.addHandler(fileHandler) + + sdfp = os.path.join('/', 'var', 'log', 'rawtools', args.module_name) # system directory file path + if not os.path.exists(sdfp): + os.makedirs(sdfp) + slfp = os.path.join(sdfp, logfile_basename) # system log file path + syslogFileHandler = logging.FileHandler(slfp) + syslogFileHandler.setFormatter(logFormatter) + syslogFileHandler.setLevel(logging.DEBUG) # always show debug statements in log file + rootLogger.addHandler(syslogFileHandler) + + consoleHandler = logging.StreamHandler() + consoleHandler.setFormatter(logFormatter) + consoleHandler.setLevel(logging_level) + rootLogger.addHandler(consoleHandler) + + logging.debug(f'Running {args.module_name} {__version__}') + logging.debug(f"Command: {args}") diff --git a/rawtools/qualitycontrol.py b/rawtools/qualitycontrol.py index 4174409..f72fab4 100644 --- a/rawtools/qualitycontrol.py +++ b/rawtools/qualitycontrol.py @@ -215,7 +215,8 @@ def get_top_down_projection(args, fp): array_buffer = arr.tobytes() pngImage = Image.new("I", arr.T.shape) pngImage.frombytes(array_buffer, "raw", "I;16") - pngImage.save(ofp) + if not args.dryrun: + pngImage.save(ofp) except Exception as err: logging.error(err) @@ -332,8 +333,9 @@ def get_side_projection(args, fp): pngImage = Image.new("I", arr.shape) logging.debug(f"pngImage.frombytes(array_buffer, 'raw', \"I;16\")") pngImage.frombytes(array_buffer, "raw", "I;16") - logging.debug(f"pngImage.save(ofp)") - pngImage.save(ofp) + if not args.dryrun: + logging.debug(f"pngImage.save(ofp)") + pngImage.save(ofp) if "step" in args and args.step: try: @@ -362,7 +364,8 @@ def get_side_projection(args, fp): draw.text((110, text_y), str(slice_index), font=font, fill=fill) # Add line draw.line((0, slice_index, 100, slice_index), fill=fill) - img.save(ofp) + if not args.dryrun: + img.save(ofp) except: raise @@ -453,7 +456,8 @@ def get_slice(args, fp): array_buffer = arr.tobytes() pngImage = Image.new("I", arr.T.shape) pngImage.frombytes(array_buffer, "raw", "I;16") - pngImage.save(ofp) + if not args.dryrun: + pngImage.save(ofp) except Exception as err: logging.error(err) sys.exit(1) @@ -532,6 +536,8 @@ def main(args): for fp in args.path: # Set working directory for file args.cwd = os.path.dirname(os.path.abspath(fp)) + if args.outdir is not None: + args.cwd = os.path.realpath(args.outdir) fp = os.path.abspath(fp) # Format file size From e3fb36685bd254da5f90869c4d10c67458313d4c Mon Sep 17 00:00:00 2001 From: Tim Parker Date: Wed, 13 Apr 2022 21:46:09 -0500 Subject: [PATCH 2/2] Update GitHub repo --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a2fdc10..b6624e5 100644 --- a/setup.py +++ b/setup.py @@ -65,7 +65,7 @@ setup_requires=setup_requirements, test_suite='tests', tests_require=test_requirements, - url='https://github.com/tparkerd/python-rawtools', + url='https://github.com/Topp-Roots-Lab/python-rawtools', version='0.3.0', zip_safe=False, )