Add script to upload packages to web server (#13545)

- Why I did it
The flow of SONiC reproducible build assumes that web packages (retrieved by curl and wget) are located in a single web file.
server. Currently there is no way to easily download the packages from the various file servers where they are currently deployed.
In this change we offer a utility to download all relevant packages and upload them to a specific file server.

- How I did it
We implemented python script that parse the various version files (generated by SONIC reproducible build compilation) and identify all relevant packages.
Later the script downloads the file and then upload them to the destination server pointed by the user.

- How to verify it
Script is running and was verified manually

Which release branch to backport (provide reason below if selected)
Feature will be added to master only

Signed-off-by: oreiss <[email protected]>

Author: orenreiss

scripts/populate_file_web_server.py

@@ -0,0 +1,295 @@
+#!/usr/bin/env python3
+
+import sys
+import os
+import time
+import argparse
+from http import HTTPStatus
+try:
+    import requests
+except ImportError:
+    print("requests module is not installed. script will fail to execute")
+
+#debug print level
+PRINT_LEVEL_ERROR         = "err"
+PRINT_LEVEL_WARN          = "warn"
+PRINT_LEVEL_INFO          = "info"
+PRINT_LEVEL_VERBOSE       = "verbose"
+
+PRINT_LEVEL_LUT = {PRINT_LEVEL_ERROR    : 1,
+                   PRINT_LEVEL_WARN     : 2,
+                   PRINT_LEVEL_INFO     : 3,
+                   PRINT_LEVEL_VERBOSE  : 4 }
+
+#return code
+RET_CODE_SUCCESS                = 0
+RET_CODE_CANNOT_CREATE_FILE     = -1
+RET_CODE_CANNOT_OPEN_FILE       = -2
+RET_CODE_HTTP_SERVER_ERROR      = -3
+RET_CODE_CANNOT_WRITE_FILE      = -4
+
+#constants
+RESOURCES_FILE_NAME       = 'versions-web'
+EXCLUDE_DIRECTORES        = ['fsroot', 'target']
+HASH_SEPARATOR            = '-'
+DEFAULT_INVALID_INPUT     = 'none'
+
+# global variables
+g_current_print_level     = PRINT_LEVEL_INFO
+
+#Script debug features (disabled by default)
+g_delete_resources_in_cache = True
+
+
+# global Classes
+class Resource:
+    def __init__(self, line, file):
+        self.file = file
+        temp=line.split("==")
+        assert(2==len(temp))
+        self.url=temp[0].strip()
+        self.hash=temp[1].strip()
+        temp=self.url.split("/")
+        assert(len(temp)>0)
+        self.name=temp[len(temp)-1]
+        #handle special scenarios
+        if 0 != self.name.count('?') == True:
+            temp = self.name.split("?")
+            self.name = temp[0]
+
+    def get_unique_name(self):
+        return self.name + HASH_SEPARATOR + self.hash
+
+    def get_url(self):
+        return self.url
+
+    def __str__(self):
+        ret_val = "Resource name: " + self.name + "\n"
+        ret_val += "File: " + self.file + "\n"
+        ret_val += "Hash: " + self.hash + "\n"
+        ret_val += "Full URL: " + self.url
+        return ret_val
+
+# Helper functions
+
+def print_msg(print_level, msg, print_in_place=False):
+    if PRINT_LEVEL_LUT[g_current_print_level] >= PRINT_LEVEL_LUT[print_level]:
+        if True == print_in_place:
+            print(msg, end='\r')
+        else:
+            print(msg)
+
+def create_dir_if_not_exist(dir):
+    if not os.path.exists(dir):
+        try:
+            os.makedirs(dir)
+        except:
+            print_msg(PRINT_LEVEL_WARN, "Cannot create directory " + dir)
+
+def delete_file_if_exist(file):
+    if os.path.exists(file):
+        try:
+            os.remove(file)
+        except:
+            print_msg(PRINT_LEVEL_WARN, "Cannot delete " + file)
+        
+# Logic functions
+        
+def generate_output_file(resources, dest_url_valid, dest_url, output_file_name):
+    try:
+        with open(output_file_name, 'w') as f:
+            for unique_name in resources.keys():
+                resource = resources[unique_name]
+                if True == dest_url_valid:
+                    line = dest_url
+                else:
+                    line = resource.get_url()
+                if line[-1] != '/':
+                    line += '/'
+                line += resource.name + "==" + resource.hash
+                f.write(line + '\n')
+    except:
+        print_msg(PRINT_LEVEL_WARN, output_file_name + " cannot be created")
+        return RET_CODE_CANNOT_CREATE_FILE
+        
+    return RET_CODE_SUCCESS
+
+def upload_resource_to_server(resource_path, resource_name, user, key, server_url):
+    url_full_path = server_url + "/" + resource_name
+
+    try:
+        f = open(resource_path, 'rb')
+    except:
+        err_print("Cannot open " + resource_path)
+        return RET_CODE_CANNOT_OPEN_FILE
+
+    headers = {'Content-type': 'application', 'Slug': resource_name}
+    response = requests.put(url_full_path, data=f,
+                            headers=headers, auth=(user, key))
+
+    f.close()
+
+    if response.status_code != HTTPStatus.CREATED.value:
+        err_print(f"HTTP request returned status code {response.status_code}, expected {HTTPStatus.CREATED.value}")
+        return RET_CODE_HTTP_SERVER_ERROR
+
+    # JSON response empty only when status code is 204
+    reported_md5 = response.json().get('checksums', {}).get('md5')
+    file_md5 = resource_name.split(HASH_SEPARATOR)[-1]
+
+    # Check if server reports checksum, if so compare reported sum and the one
+    # specified in filename
+    if reported_md5 != None and reported_md5 != file_md5:
+        print_msg(PRINT_LEVEL_WARN, f"Server reported file's chsum {reported_md5}, expected {file_md5}")
+
+    
+    return RET_CODE_SUCCESS
+
+def download_external_resouce(resource, cache_path):
+    resource_path_in_cache = cache_path + os.sep + resource.get_unique_name()
+
+    r = requests.get(resource.get_url(), allow_redirects=True)
+
+    try:
+        f = open(resource_path_in_cache, 'wb')
+        f.write(r.content)
+        f.close()
+    except:
+        print_msg(PRINT_LEVEL_ERROR, "Cannot write " + resource_path_in_cache + " to cache")
+        resource_path_in_cache = "" #report error
+
+    return resource_path_in_cache
+
+def get_resources_list(resource_files_list):
+    resource_list = list()
+
+    for file_name in resource_files_list:
+        try:
+            with open(file_name, 'r') as f:
+                for line in f:
+                    resource_list.append(Resource(line, file_name))
+        except:
+            print_msg(PRINT_LEVEL_WARN, file_name + " cannot be opened")
+
+    return resource_list
+
+def filter_out_dir(subdir):
+    ret_val = False
+
+    for exclude in EXCLUDE_DIRECTORES:
+        if exclude in subdir.split(os.sep):
+            ret_val = True
+            break
+
+    return ret_val
+
+def get_resource_files_list(serach_path):
+    resource_files_list = list()
+
+    for subdir, dirs, files in os.walk(serach_path):
+        for file in files:
+            if False == filter_out_dir(subdir) and RESOURCES_FILE_NAME == file:
+                file_full_path = os.path.join(subdir, file)
+                print_msg(PRINT_LEVEL_VERBOSE, "Found resource file :" + file_full_path)
+                resource_files_list.append(file_full_path)
+
+    return resource_files_list
+
+def parse_args():
+    parser = argparse.ArgumentParser(description='Various pre-steps for build compilation')
+
+    parser.add_argument('-s', '--source', default=".",
+                        help='Search path for ' + RESOURCES_FILE_NAME + ' files')
+
+    parser.add_argument('-c', '--cache', default="." + os.sep + "tmp",
+                        help='Path to cache for storing content before uploading to server')
+
+    parser.add_argument('-p', '--print', default=PRINT_LEVEL_INFO, 
+                        choices=[PRINT_LEVEL_ERROR, PRINT_LEVEL_WARN, PRINT_LEVEL_INFO, PRINT_LEVEL_VERBOSE],
+                        help='Print level verbosity')
+                        
+    parser.add_argument('-o', '--output', default=DEFAULT_INVALID_INPUT,
+                        help='Output file name to hold the list of packages')                    
+                        
+    parser.add_argument('-u', '--user', default=DEFAULT_INVALID_INPUT,
+                        help='User for server authentication')
+
+    parser.add_argument('-k', '--key', default=DEFAULT_INVALID_INPUT,
+                        help='API key server authentication')
+
+    parser.add_argument('-d', '--dest', default=DEFAULT_INVALID_INPUT,
+                        help='URL for destination web file server')
+
+    return parser.parse_args()
+
+def main():
+    global g_current_print_level 
+    ret_val = RET_CODE_SUCCESS
+    resource_counter = 0.0
+    resource_dict = dict()
+
+    args = parse_args()
+       
+    g_current_print_level = args.print
+    
+    resource_files_list = get_resource_files_list(args.source)
+
+    resource_list = get_resources_list(resource_files_list)
+
+    #remove duplications 
+    for resource in resource_list:
+        unique_name = resource.get_unique_name()
+        if not unique_name in resource_dict.keys():
+            resource_dict[unique_name] = resource
+
+    print_msg(PRINT_LEVEL_INFO, "Found " + str(len(resource_files_list)) + " version files and " + str(len(resource_dict.keys())) + " unique resources")
+    
+    if args.dest != DEFAULT_INVALID_INPUT:
+        upload_files_to_server = True
+        print_msg(PRINT_LEVEL_INFO, "Upload files to URL - " + args.dest)
+    else:
+        upload_files_to_server = False
+        print_msg(PRINT_LEVEL_INFO, "Skipping files upload to server")       
+
+    #create cache directory if not exist
+    create_dir_if_not_exist(args.cache)
+
+    #download content to cache and then upload to web server
+    for unique_name in resource_dict.keys():
+
+        resource = resource_dict[unique_name]
+
+        print_msg(PRINT_LEVEL_VERBOSE, resource)
+
+        resource_counter += 1.0
+
+        #download content to cache
+        file_in_cache = download_external_resouce(resource, args.cache)
+        
+        if "" == file_in_cache:
+            return RET_CODE_CANNOT_WRITE_FILE
+        
+        if True == upload_files_to_server:
+            #upload content to web server
+            ret_val = upload_resource_to_server(file_in_cache, unique_name, args.user, args.key, args.dest)
+            if ret_val != RET_CODE_SUCCESS:
+                return ret_val
+        
+        if True == g_delete_resources_in_cache:
+            delete_file_if_exist(file_in_cache)
+
+        print_msg(PRINT_LEVEL_INFO, "Downloading Data. Progress " + str(int(100.0*resource_counter/len(resource_dict.keys()))) + "%", True) #print progress bar
+    
+    # generate version output file as needed
+    if args.output != DEFAULT_INVALID_INPUT:
+        ret_val = generate_output_file(resource_dict, upload_files_to_server, args.dest, args.output)
+        print_msg(PRINT_LEVEL_INFO, "Generate output file " + args.output)
+        
+    return ret_val
+
+#    Entry function                                                
+if __name__ == '__main__':
+
+    ret_val = main()
+
+    sys.exit(ret_val)