From bb77c5c2b086b47d2bbf0d4d30af44c63c3d3d3c Mon Sep 17 00:00:00 2001 From: David Shupe Date: Fri, 8 Dec 2023 16:57:14 -0800 Subject: [PATCH 1/7] changes to work with recent versions of firefly_client * add pause before uploading table * add FIREFLY_URL environment variable as the default * print a message while files are being pattern-matched * reformat single-quotes to double-quotes --- examples/filetable.py | 92 +++++++++++++++++++++++++------------------ 1 file changed, 53 insertions(+), 39 deletions(-) diff --git a/examples/filetable.py b/examples/filetable.py index 63e5734..5dc7cc0 100755 --- a/examples/filetable.py +++ b/examples/filetable.py @@ -7,6 +7,7 @@ import firefly_client + def filetable_to_firefly(ffclient, topdir, pattern, recursive=True): """Upload table of files matching a pattern to a FireflyClient @@ -15,80 +16,93 @@ def filetable_to_firefly(ffclient, topdir, pattern, recursive=True): ffclient: firefly_client.FireflyClient Instance of FireflyClient connected to a Firefly server - topdir: 'str' + topdir: "str" pathname for directory to search - pattern: 'str' - filename pattern to search for, e.g. '*.fits' + pattern: "str" + filename pattern to search for, e.g. "*.fits" recursive: `bool` Search all subdirectories recursively (default=True) Returns: -------- - tbl_val: 'str' + tbl_val: "str" Descriptor of table that was uploaded to the Firefly server metadict: `dict` Dictionary of metadata items """ - filelist = glob(topdir+'/**/' + pattern, recursive=recursive) - metadict = {'datasource': 'path'} - with tempfile.NamedTemporaryFile(mode='w+t', - delete=False, - suffix='.csv') as fd: + filelist = glob(topdir + "/**/" + pattern, recursive=recursive) + metadict = {"datasource": "path"} + with tempfile.NamedTemporaryFile(mode="w+t", delete=False, suffix=".csv") as fd: csv_writer = csv.writer(fd) - csv_writer.writerow(['number','name','path']) + csv_writer.writerow(["number", "name", "path"]) for i, path in enumerate(filelist): - csv_writer.writerow([i, os.path.basename(path), - 'file://'+path]) + csv_writer.writerow([i, os.path.basename(path), "file://" + path]) tbl_val = ffclient.upload_file(fd.name) os.remove(fd.name) - return(tbl_val, metadict) + return (tbl_val, metadict) # Sample application def main(): import argparse - parser = argparse.ArgumentParser(description=""" + + parser = argparse.ArgumentParser( + description=""" Display a table of files in a Firefly window - """) - parser.add_argument('topdir', help='top-level directory to search') - parser.add_argument('pattern', help='filename pattern for search') - parser.add_argument('--norecursion', help='do not recursively search topdir', - action='store_true') - parser.add_argument('--host', help='host address for Firefly', - default='localhost') - parser.add_argument('--port', help='port for Firefly', default='8080') - parser.add_argument('--base', help='base.for Firefly', default='firefly') - parser.add_argument('--channel', help='channel name for websocket', - default=None) - parser.add_argument('--printurl', help='print browser url instead of' + - ' attempting to launch browser', action='store_true') + """ + ) + parser.add_argument("topdir", help="top-level directory to search") + parser.add_argument("pattern", help="filename pattern for search") + parser.add_argument( + "--norecursion", help="do not recursively search topdir", action="store_true" + ) + parser.add_argument( + "--firefly_url", help="URL for Firefly server", default=os.getenv("FIREFLY_URL") + ) + parser.add_argument("--channel", help="channel name for websocket", default=None) + parser.add_argument( + "--printurl", + help="print browser url instead of" + " attempting to launch browser", + action="store_true", + ) args = parser.parse_args() topdir = args.topdir pattern = args.pattern - host = args.host - port = args.port - basedir = args.base + firefly_url = args.firefly_url channel = args.channel printurl = args.printurl + launch_browser = True + if printurl: + launch_browser = False recursion = not args.norecursion - fc = FireflyClient.make_client(url='{}:{}'.format(host, port), channel_override=channel, html_file='slate.html') + fc = firefly_client.FireflyClient.make_client( + url=firefly_url, + channel_override=channel, + html_file="slate.html", + launch_browser=launch_browser, + ) + if printurl: + print("Firefly URL is {}".format(fc.get_firefly_url())) + + print("Searching for files...", end="") + tbl_val, metainfo = filetable_to_firefly(fc, topdir, pattern, recursive=recursion) + print("done.") + if printurl: - print('Firefly url is {}'.format(fc.get_firefly_url())) + input("Press Enter after your browser is open to the Firefly URL...") - tbl_val, metainfo = filetable_to_firefly(fc, topdir, pattern, - recursive=recursion) - r = fc.add_cell(0, 0, 1, 2, 'tables', 'main') + r = fc.add_cell(0, 0, 1, 2, "tables", "main") fc.show_table(tbl_val, meta=metainfo) - r = fc.add_cell(0, 1, 1, 2, 'tableImageMeta', 'image-meta') - fc.show_image_metadata(viewer_id=r['cell_id']) + r = fc.add_cell(0, 1, 1, 2, "tableImageMeta", "image-meta") + fc.show_image_metadata(viewer_id=r["cell_id"]) -if __name__ == '__main__': - main() +if __name__ == "__main__": + main() From c8af4ec34b9bd9065941d05f03bdca6219ed4c6f Mon Sep 17 00:00:00 2001 From: David Shupe Date: Wed, 13 Dec 2023 15:09:26 -0800 Subject: [PATCH 2/7] add /external to the allowed search path --- examples/filetable.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/filetable.py b/examples/filetable.py index 5dc7cc0..cbc669d 100755 --- a/examples/filetable.py +++ b/examples/filetable.py @@ -40,7 +40,10 @@ def filetable_to_firefly(ffclient, topdir, pattern, recursive=True): csv_writer = csv.writer(fd) csv_writer.writerow(["number", "name", "path"]) for i, path in enumerate(filelist): - csv_writer.writerow([i, os.path.basename(path), "file://" + path]) + # Docker Firefly allows uploads from /external + csv_writer.writerow([i, + os.path.basename(path), + "file:///external" + path]) tbl_val = ffclient.upload_file(fd.name) os.remove(fd.name) From 3f3c03bc568fbb993623e16b7002d2df60230155 Mon Sep 17 00:00:00 2001 From: David Shupe Date: Wed, 13 Dec 2023 15:13:25 -0800 Subject: [PATCH 3/7] sort the list of file paths --- examples/filetable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/filetable.py b/examples/filetable.py index cbc669d..d778a4b 100755 --- a/examples/filetable.py +++ b/examples/filetable.py @@ -34,7 +34,7 @@ def filetable_to_firefly(ffclient, topdir, pattern, recursive=True): Dictionary of metadata items """ - filelist = glob(topdir + "/**/" + pattern, recursive=recursive) + filelist = sorted(glob(topdir + "/**/" + pattern, recursive=recursive)) metadict = {"datasource": "path"} with tempfile.NamedTemporaryFile(mode="w+t", delete=False, suffix=".csv") as fd: csv_writer = csv.writer(fd) From acd9042c9a38596cbfb642382f022f821558c28d Mon Sep 17 00:00:00 2001 From: David Shupe Date: Wed, 17 Jan 2024 16:20:11 -0800 Subject: [PATCH 4/7] Apply suggestions from code review Co-authored-by: Jaladh Singhal --- examples/filetable.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/examples/filetable.py b/examples/filetable.py index d778a4b..bb4402f 100755 --- a/examples/filetable.py +++ b/examples/filetable.py @@ -80,9 +80,7 @@ def main(): firefly_url = args.firefly_url channel = args.channel printurl = args.printurl - launch_browser = True - if printurl: - launch_browser = False + launch_browser = False if printurl else True recursion = not args.norecursion fc = firefly_client.FireflyClient.make_client( @@ -99,7 +97,7 @@ def main(): print("done.") if printurl: - input("Press Enter after your browser is open to the Firefly URL...") + input("Press Enter after you have opened the Firefly URL printed above...") r = fc.add_cell(0, 0, 1, 2, "tables", "main") fc.show_table(tbl_val, meta=metainfo) From 15e05b41dda08a4b739d7e927da86ffea621c61a Mon Sep 17 00:00:00 2001 From: David Shupe Date: Wed, 17 Jan 2024 16:38:45 -0800 Subject: [PATCH 5/7] add arguments for sorting and path prefix --- examples/filetable.py | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/examples/filetable.py b/examples/filetable.py index bb4402f..0b85bcf 100755 --- a/examples/filetable.py +++ b/examples/filetable.py @@ -8,7 +8,9 @@ import firefly_client -def filetable_to_firefly(ffclient, topdir, pattern, recursive=True): +def filetable_to_firefly( + ffclient, topdir, pattern, path_prefix="", recursive=True, sort=False +): """Upload table of files matching a pattern to a FireflyClient Parameters: @@ -22,9 +24,17 @@ def filetable_to_firefly(ffclient, topdir, pattern, recursive=True): pattern: "str" filename pattern to search for, e.g. "*.fits" + path_prefix: "str" + string to prepend to each file path after "file://" + for example, specify "/external" for Firefly-in-Docker + (default="") + recursive: `bool` Search all subdirectories recursively (default=True) + sort: `bool` + Sort the file paths (default=False) + Returns: -------- tbl_val: "str" @@ -34,16 +44,18 @@ def filetable_to_firefly(ffclient, topdir, pattern, recursive=True): Dictionary of metadata items """ - filelist = sorted(glob(topdir + "/**/" + pattern, recursive=recursive)) + filelist = glob(topdir + "/**/" + pattern, recursive=recursive) + if sort: + filelist = sorted(filelist) metadict = {"datasource": "path"} with tempfile.NamedTemporaryFile(mode="w+t", delete=False, suffix=".csv") as fd: csv_writer = csv.writer(fd) csv_writer.writerow(["number", "name", "path"]) for i, path in enumerate(filelist): # Docker Firefly allows uploads from /external - csv_writer.writerow([i, - os.path.basename(path), - "file:///external" + path]) + csv_writer.writerow( + [i, os.path.basename(path), "file://" + path_prefix + path] + ) tbl_val = ffclient.upload_file(fd.name) os.remove(fd.name) @@ -61,9 +73,16 @@ def main(): ) parser.add_argument("topdir", help="top-level directory to search") parser.add_argument("pattern", help="filename pattern for search") + parser.add_argument( + "--path_prefix", + help="string to prepend to file paths\n" + + "e.g. specify '/external' for Firefly-in-Docker", + default="", + ) parser.add_argument( "--norecursion", help="do not recursively search topdir", action="store_true" ) + parser.add_argument("--sort", help="sort the file paths", action="store_true") parser.add_argument( "--firefly_url", help="URL for Firefly server", default=os.getenv("FIREFLY_URL") ) @@ -82,6 +101,8 @@ def main(): printurl = args.printurl launch_browser = False if printurl else True recursion = not args.norecursion + sort = args.sort + path_prefix = args.path_prefix fc = firefly_client.FireflyClient.make_client( url=firefly_url, @@ -93,7 +114,9 @@ def main(): print("Firefly URL is {}".format(fc.get_firefly_url())) print("Searching for files...", end="") - tbl_val, metainfo = filetable_to_firefly(fc, topdir, pattern, recursive=recursion) + tbl_val, metainfo = filetable_to_firefly( + fc, topdir, pattern, path_prefix=path_prefix, recursive=recursion, sort=sort + ) print("done.") if printurl: From 2183bcc2d63b89718e521efe883f7dd895303d07 Mon Sep 17 00:00:00 2001 From: David Shupe Date: Wed, 17 Jan 2024 17:06:36 -0800 Subject: [PATCH 6/7] put extended description about running Firefly server locally --- examples/filetable.py | 38 +++++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/examples/filetable.py b/examples/filetable.py index 0b85bcf..4fd8895 100755 --- a/examples/filetable.py +++ b/examples/filetable.py @@ -1,13 +1,26 @@ #!/usr/bin/env python +from argparse import ArgumentParser, HelpFormatter import csv from glob import glob import os import tempfile +import textwrap import firefly_client +# From https://stackoverflow.com/a/64102901 +class RawFormatter(HelpFormatter): + def _fill_text(self, text, width, indent): + return "\n".join( + [ + textwrap.fill(line, width) + for line in textwrap.indent(textwrap.dedent(text), indent).splitlines() + ] + ) + + def filetable_to_firefly( ffclient, topdir, pattern, path_prefix="", recursive=True, sort=False ): @@ -64,19 +77,30 @@ def filetable_to_firefly( # Sample application def main(): - import argparse - - parser = argparse.ArgumentParser( + parser = ArgumentParser( description=""" - Display a table of files in a Firefly window - """ + Display a table of files in a Firefly window. + + Note that you must be running a Firefly server that is + local to the data. + If running Firefly via Docker, note that you can mount your + disk area onto /external. + + Sample command for running the Firefly server: + docker run -p 8090:8080 -e "MAX_JVM_SIZE=64G" \\ + -e "LOG_FILE_TO_CONSOLE=firefly.log" --rm --name ncmds_firefly \\ + -v /neoswork:/external/neoswork ipac/firefly:latest + """, + formatter_class=RawFormatter, ) parser.add_argument("topdir", help="top-level directory to search") parser.add_argument("pattern", help="filename pattern for search") parser.add_argument( "--path_prefix", - help="string to prepend to file paths\n" - + "e.g. specify '/external' for Firefly-in-Docker", + help=textwrap.dedent( + """string to prepend to file paths, + e.g. specify '/external' for Firefly-in-Docker""" + ), default="", ) parser.add_argument( From 073e279d7eec06b938a57c70152f5ff89458ae50 Mon Sep 17 00:00:00 2001 From: David Shupe Date: Wed, 17 Jan 2024 17:10:48 -0800 Subject: [PATCH 7/7] use epilog for note about /external and running Firefly --- examples/filetable.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/filetable.py b/examples/filetable.py index 4fd8895..34ae96f 100755 --- a/examples/filetable.py +++ b/examples/filetable.py @@ -83,6 +83,8 @@ def main(): Note that you must be running a Firefly server that is local to the data. + """, + epilog=""" If running Firefly via Docker, note that you can mount your disk area onto /external.