Merge pull request #75 from UAL-RE/66-keep-track-of-how-many-items-were-successfully-bagged-new

Feat: Keep track of items successfully processed (Issue #66)

Author: zoidy

Log.py

@@ -23,6 +23,8 @@ def __init__(self, config):
         self.file_path = log_location + file_name
 
         self.ansi_terminal = _check_ansi()
+        self.warnings_count = 0
+        self.errors_count = 0
 
     def log_config(self, in_terminal: bool = False):
         if (in_terminal):
@@ -37,13 +39,15 @@ def log_config(self, in_terminal: bool = False):
     def show_log_in_terminal(self, type, message, stop_script=False):
         # Show log in terminal
         self.log_config(True)
+        self._count_errorwarning(type)
         self.message(type, message)
         if (stop_script is True):
             exit()
 
     def write_log_in_file(self, type, message, show_in_terminal=False, stop_script=False):
         # Show log in file
         self.log_config(False)
+        self._count_errorwarning(type)
         if (show_in_terminal is True):
             print(datetime.now().strftime("%Y-%m-%d %H:%M:%S,%f")[:-3] + ":" + self._format_messagetype_ansi(type.upper()) + ": " + message)
         self.message(type, message)
@@ -72,6 +76,15 @@ def message(self, type, message):
             logger.error(message)
             del logger
 
+    def _count_errorwarning(self, msgtype):
+        '''
+        Counts how many times a message type (string) of warning or error is passed in
+        '''
+        if msgtype.lower() == 'warning':
+            self.warnings_count += 1
+        if msgtype.lower() == 'error':
+            self.errors_count += 1
+
     def _format_messagetype_ansi(self, type):
         '''
         Returns a colorized version of the given message type string. If no ANSI support is detected, the same string is returned unchanged.

app.py

@@ -143,8 +143,14 @@ def main():
                           True)
     article_obj = Article(config_file_path, log, args.ids)
     article_data = article_obj.get_articles()
+
+    articles_count = 0
+    articles_versions_count = 0
+    for i, (k, v) in enumerate(article_data.items()):
+        articles_count += 1
+        articles_versions_count += len(v)
     log.write_log_in_file('info',
-                          f"Total articles fetched: {len(article_data)}.",
+                          f"Total articles fetched: {len(article_data)}. Total articles versions fetched: {articles_versions_count}.",
                           True)
     print(" ")
 
@@ -153,15 +159,45 @@ def main():
                           True)
     collection_obj = Collection(config_file_path, log, args.ids)
     collection_data = collection_obj.get_collections()
+
+    collections_count = 0
+    collections_versions_count = 0
+    for i, (k, v) in enumerate(collection_data.items()):
+        collections_count += 1
+        collections_versions_count += len(v['versions'])
     log.write_log_in_file('info',
-                          f"Total collections fetched: {len(collection_data)}.",
+                          f"Total collections fetched: {collections_count}. Total collections versions fetched: {collections_versions_count}.",
                           True)
     print(" ")
 
     # Start articles processing after completing fetching data from API
-    article_obj.process_articles(article_data, article_obj.total_all_articles_file_size)
+    processed_articles_versions_count = article_obj.process_articles(article_data)
 
     # Start collections processing after completing fetching data from API and articles processing.
-    collection_obj.process_collections(collection_data)
+    processed_collections_versions_count = collection_obj.process_collections(collection_data)
+
+    log.write_log_in_file('info',
+                          "Total articles versions processed/fetched: \t\t\t"
+                          + f'{processed_articles_versions_count} / {articles_versions_count}',
+                          True)
+    log.write_log_in_file('info',
+                          "Total processed articles bags already in preservation storage: \t"
+                          + f'{article_obj.processor.duplicate_bag_in_preservation_storage_count}',
+                          True)
+    log.write_log_in_file('info',
+                          "Total collections versions processed/fetched: \t\t\t"
+                          + f'{processed_collections_versions_count} / {collections_versions_count}',
+                          True)
+    log.write_log_in_file('info',
+                          "Total processed collections bags already in preservation storage: "
+                          + f'{collection_obj.processor.duplicate_bag_in_preservation_storage_count}',
+                          True)
 
-    log.write_log_in_file('info', "ReBACH script has successfully finished.", True, True)
+    if processed_articles_versions_count != articles_versions_count or processed_collections_versions_count != collections_versions_count:
+        log.write_log_in_file('warning',
+                              'The number of articles versions or collections versions sucessfully processed is different'
+                              + ' than the number fetched. Check the log for details.', True)
+
+    log.write_log_in_file('info',
+                          f"ReBACH finished with {log.warnings_count} warnings and {log.errors_count} errors",
+                          True)

figshare/Article.py

@@ -738,10 +738,13 @@ def find_matched_articles(self, articles):
 
         self.logs.write_log_in_file("info", f"Total matched unique articles: {len(set(matched_articles))}.", True)
         self.logs.write_log_in_file("info", f"Total unmatched unique articles: {len(set(unmatched_articles))}.", True)
-
         self.logs.write_log_in_file("info", f"Total matched article versions: {no_matched}.", True)
         self.logs.write_log_in_file("info", f"Total unmatched article versions: {len(self.article_non_match_info)}.", True)
 
+        if len(set(unmatched_articles)) > 0 or len(self.article_non_match_info) > 0:
+            self.logs.write_log_in_file("warning", "There were unmatched articles or article versions."
+                                        + f"Check {self.curation_storage_location} for each of the unmatched items.", True)
+
         return article_data
 
     """
@@ -761,7 +764,7 @@ def __can_copy_files(self, version_data):
         return copy_files
 
     """
-    Final process for matched articles.
+    Final process for matched articles. Returns True if succeeded.
     """
     def __final_process(self, check_files, copy_files, check_dir, version_data, folder_name, version_no, value_pre_process):
         success = True
@@ -798,14 +801,19 @@ def __final_process(self, check_files, copy_files, check_dir, version_data, fold
                         self.logs.write_log_in_file("error",
                                                     f"{version_data['id']} version {version_data['version']} - Post-processing script failed.",
                                                     True)
+                        success = False
+                    else:
+                        success = True
                 else:
                     self.logs.write_log_in_file("info",
                                                 f"No further processing for {version_data['id']} version {version_data['version']} due to errors.",
                                                 True)
+                    success = False
             else:
                 # if download process has any errors then delete complete folder
                 self.logs.write_log_in_file("info", "Download process had an error so complete folder is being deleted.", True)
                 self.delete_folder(check_dir)
+                success = False
         else:
             if check_files or copy_files:
                 if success:
@@ -815,12 +823,18 @@ def __final_process(self, check_files, copy_files, check_dir, version_data, fold
                         self.logs.write_log_in_file("error",
                                                     f"{version_data['id']} version {version_data['version']} - Post-processing script failed.",
                                                     True)
+                        success = False
+                    else:
+                        success = True
                 else:
                     self.logs.write_log_in_file("info",
                                                 f"No further processing for {version_data['id']} version {version_data['version']} due to errors.",
                                                 True)
+                    success = False
             else:
                 self.logs.write_log_in_file("error", "Unexpected condidion in final processing. No further actions taken.", True)
+                success = False
+        return success
 
     """
     Called before articles processing.
@@ -839,9 +853,10 @@ def __initial_process(self):
         return curation_storage_location
 
     """
-    Process all articles after fetching from API.
+    Process all articles after fetching from API. Returns the number of successfully processed articles.
     """
-    def process_articles(self, articles, total_file_size):
+    def process_articles(self, articles):
+        processed_count = 0
         curation_storage_location = self.__initial_process()
         self.logs.write_log_in_file("info", "Finding matched articles.", True)
         article_data = self.find_matched_articles(articles)
@@ -859,7 +874,7 @@ def process_articles(self, articles, total_file_size):
 
         required_space = curation_folder_size + self.total_all_articles_file_size
 
-        self.logs.write_log_in_file("info", f"Total size of aritcles to be processed: {self.total_all_articles_file_size} bytes", True)
+        self.logs.write_log_in_file("info", f"Total size of articles to be processed: {self.total_all_articles_file_size} bytes", True)
         self.logs.write_log_in_file("info", f"Total size of the curated folders for the matched articles: {curation_folder_size} bytes", True)
         self.logs.write_log_in_file("info", f"Total space required: {required_space} bytes", True)
 
@@ -905,19 +920,22 @@ def process_articles(self, articles, total_file_size):
                                         self.logs.write_log_in_file("error", f"{version_data['id']} version {version_data['version']} - "
                                                                     + "Post-processing script error found.", True)
                                     break
+
                             # end check main folder exists in preservation storage.
                             # check required files exist in curation UAL_RDM folder
                             self.logs.write_log_in_file("info", "Checking required files exist in associated curation "
                                                         + f"folder {curation_storage_location}.", True)
                             copy_files = self.__can_copy_files(version_data)
-                            self.__final_process(check_files, copy_files, check_dir, version_data, folder_name, version_no, value_pre_process)
+                            if self.__final_process(check_files, copy_files, check_dir, version_data, folder_name, version_no, value_pre_process):
+                                processed_count += 1
                         else:
                             self.logs.write_log_in_file("error", "Pre-processing script failed. Running post-processing script.", True)
                             # call post process script function for each matched item.
                             value_post_process = self.processor.post_process_script_function("Article", check_dir, value_pre_process)
                             if (value_post_process != 0):
                                 self.logs.write_log_in_file("error", f"{version_data['id']} version {version_data['version']} - "
                                                             + "Post-processing script failed.", True)
+        return processed_count
 
     """
     Preservation and Curation directory access check while processing.

figshare/Collection.py

@@ -235,10 +235,11 @@ def get_article_api_url(self, collection):
         return coll_articles_api
 
     """
-    Function to process collections and its articles with collection versions.
+    Function to process collections and its articles with collection versions. Returns the number of successfully processed collections.
     :param collections object
     """
     def process_collections(self, collections):
+        processed_count = 0
         self.logs.write_log_in_file("info", "Processing collections.", True)
         for collection in collections:
             data = collections[collection]
@@ -251,12 +252,15 @@ def process_collections(self, collections):
                 author_name = re.sub("[^A-Za-z0-9]", "_", version['authors'][0]['full_name'])
                 folder_name = str(collection) + "_" + version_no + "_" + author_name + "_" + version_md5 + "/" + version_no + "/METADATA"
                 version["articles"] = articles
-                self.logs.write_log_in_file("info", f"Processing collection {collection} version {version['version']}.", True)
+                self.logs.write_log_in_file("info", f"------- Processing collection {collection} version {version['version']}.", True)
                 self.__save_json_in_metadata(collection, version, folder_name)
                 collection_preservation_path = self.preservation_storage_location + os.path.basename(os.path.dirname(os.path.dirname(folder_name)))
                 value_post_process = self.processor.post_process_script_function("Collection", collection_preservation_path)
                 if (value_post_process != 0):
                     self.logs.write_log_in_file("error", f"collection {collection} - post-processing script failed.", True)
+                else:
+                    processed_count += 1
+        return processed_count
 
     """
     Save json data for each collection version in related directory

figshare/Integration.py

@@ -19,17 +19,17 @@ def __init__(self, config: Config, log: Log):
         """
         self._config = config
         self._rebachlogger = log
+        self.duplicate_bag_in_preservation_storage_count = 0
 
     """
     Post-processing script command function.
     """
     def post_process_script_function(self, *args):
         """
-        Execute a post-processing script on an article or collection package and return the result.
-
         If the 'post_process_script_command' value in the configuration file is set to 'Bagger', this function
-        will execute the post-processing script internally. Otherwise, it will expect the value to be a path
-        to an external script, which will be called.
+        will execute. Returns the return code of bagger.
+
+        Otherwise, The code will expect the value to be a path to an external script, which will be called *TODO*.
 
         :param args: Variable-length arguments passed to the function.
                      args[0]: 'Article' or 'Collection' to indicate whether the function is called from an Article or Collection.
@@ -90,13 +90,13 @@ def post_process_script_function(self, *args):
                 self._rebachlogger.write_log_in_file("info", f"Exit code: {status}.", True)
                 if (status == 0):
                     self._rebachlogger.write_log_in_file("info", f"Preservation package '{preservation_package_name}' processed successfully", True)
-                    return 0
                 elif (status == 3):
+                    # code 3 is special since we don't want to cause the calling code to interpret duplicates as an error since it will happen a lot
                     self._rebachlogger.write_log_in_file("warning", f"'{preservation_package_name}' already exists in "
                                                          + f"{config['Wasabi']['host']}/{config['Wasabi']['bucket']}. File not uploaded.", True)
-                    return 0
-                else:
-                    return status
+                    self.duplicate_bag_in_preservation_storage_count += 1
+                    status = 0
+                return status
         else:
             self._rebachlogger.write_log_in_file("info",
                                                  f"[not implemented] Executing post-processing script Command: {post_process_script_command}.", True)