|
| 1 | +#!/usr/bin/python3 |
| 2 | +# |
| 3 | +# Copyright (C) Maarten Bosmans 2011 |
| 4 | +# |
| 5 | +# The contents of this file are subject to the Mozilla Public License Version 1.1; you may not use this file except in |
| 6 | +# compliance with the License. You may obtain a copy of the License at http://www.mozilla.org/MPL/ |
| 7 | +# |
| 8 | +from urllib.request import urlretrieve, urlopen |
| 9 | +from logging import warning, error |
| 10 | +import logging |
| 11 | +import os.path |
| 12 | +import sys |
| 13 | + |
| 14 | +_packages = [] |
| 15 | + |
| 16 | +_scriptDirectory = os.path.dirname(os.path.realpath(__file__)) |
| 17 | +_packageCacheDirectory = os.path.join(_scriptDirectory, 'cache', 'package') |
| 18 | +_repositoryCacheDirectory = os.path.join(_scriptDirectory, 'cache', 'repository') |
| 19 | +_extractedCacheDirectory = os.path.join(_scriptDirectory, 'cache', 'extracted') |
| 20 | +_extractedFilesDirectory = _scriptDirectory |
| 21 | + |
| 22 | + |
| 23 | +def OpenRepository(repositoryLocation): |
| 24 | + from xml.etree.cElementTree import parse as xmlparse |
| 25 | + global _packages |
| 26 | + # Check repository for latest primary.xml |
| 27 | + with urlopen(repositoryLocation + 'repodata/repomd.xml') as metadata: |
| 28 | + doctree = xmlparse(metadata) |
| 29 | + xmlns = 'http://linux.duke.edu/metadata/repo' |
| 30 | + for element in doctree.findall('{%s}data'%xmlns): |
| 31 | + if element.get('type') == 'primary': |
| 32 | + primaryUrl = element.find('{%s}location'%xmlns).get('href') |
| 33 | + # Make sure all the cache directories exist |
| 34 | + for dir in _packageCacheDirectory, _repositoryCacheDirectory, _extractedCacheDirectory: |
| 35 | + try: |
| 36 | + os.makedirs(dir) |
| 37 | + except OSError: pass |
| 38 | + # Download repository metadata (only if not already in cache) |
| 39 | + primaryFilename = os.path.join(_repositoryCacheDirectory, os.path.splitext(os.path.basename(primaryUrl))[0]) |
| 40 | + if not os.path.exists(primaryFilename): |
| 41 | + warning('Dowloading repository data') |
| 42 | + with urlopen(repositoryLocation + primaryUrl) as primaryGzFile: |
| 43 | + import io, gzip |
| 44 | + primaryGzString = io.BytesIO(primaryGzFile.read()) #3.2: use gzip.decompress |
| 45 | + with gzip.GzipFile(fileobj=primaryGzString) as primaryGzipFile: |
| 46 | + with open(primaryFilename, 'wb') as primaryFile: |
| 47 | + primaryFile.writelines(primaryGzipFile) |
| 48 | + elements = xmlparse(primaryFilename) |
| 49 | + # Parse package list from XML |
| 50 | + xmlns = 'http://linux.duke.edu/metadata/common' |
| 51 | + rpmns = 'http://linux.duke.edu/metadata/rpm' |
| 52 | + _packages = [{ |
| 53 | + 'name': p.find('{%s}name'%xmlns).text, |
| 54 | + 'arch': p.find('{%s}arch'%xmlns).text, |
| 55 | + 'buildtime': int(p.find('{%s}time'%xmlns).get('build')), |
| 56 | + 'url': repositoryLocation + p.find('{%s}location'%xmlns).get('href'), |
| 57 | + 'filename': os.path.basename(p.find('{%s}location'%xmlns).get('href')), |
| 58 | + 'provides': {provides.attrib['name'] for provides in p.findall('{%s}format/{%s}provides/{%s}entry'%(xmlns,rpmns,rpmns))}, |
| 59 | + 'requires': {req.attrib['name'] for req in p.findall('{%s}format/{%s}requires/{%s}entry'%(xmlns,rpmns,rpmns))} |
| 60 | + } for p in elements.findall('{%s}package'%xmlns)] |
| 61 | + |
| 62 | + |
| 63 | +def _findPackage(packageName, srcpkg=False): |
| 64 | + filter_func = lambda p: (p['name'] == packageName or p['filename'] == packageName) and p['arch'] == ('src' if srcpkg else 'noarch') |
| 65 | + sort_func = lambda p: p['buildtime'] |
| 66 | + packages = sorted([p for p in _packages if filter_func(p)], key=sort_func, reverse=True) |
| 67 | + if len(packages) == 0: |
| 68 | + return None |
| 69 | + if len(packages) > 1: |
| 70 | + error('multiple packages found for %s:', packageName) |
| 71 | + for p in packages: |
| 72 | + error(' %s', p['filename']) |
| 73 | + return packages[0] |
| 74 | + |
| 75 | + |
| 76 | +def _checkPackageRequirements(package, packageNames): |
| 77 | + allProviders = set() |
| 78 | + for requirement in package['requires']: |
| 79 | + providers = {p['name'] for p in _packages if requirement in p['provides']} |
| 80 | + if len(providers & packageNames) == 0: |
| 81 | + if len(providers) == 0: |
| 82 | + error('Package %s requires %s, not provided by any package', package['name'], requirement) |
| 83 | + else: |
| 84 | + warning('Package %s requires %s, provided by: %s', package['name'], requirement, ','.join(providers)) |
| 85 | + allProviders.add(providers.pop()) |
| 86 | + return allProviders |
| 87 | + |
| 88 | + |
| 89 | +def packagesDownload(packageNames, withDependencies=False, srcpkg=False): |
| 90 | + from fnmatch import fnmatchcase |
| 91 | + packageNames_new = {pn for pn in packageNames if pn.endswith('.rpm')} |
| 92 | + for packageName in packageNames - packageNames_new: |
| 93 | + matchedpackages = {p['name'] for p in _packages if fnmatchcase(p['name'].replace('mingw32-', '').replace('mingw64-', ''), packageName) and p['arch'] == ('src' if srcpkg else 'noarch')} |
| 94 | + packageNames_new |= matchedpackages if len(matchedpackages) > 0 else {packageName} |
| 95 | + packageNames = list(packageNames_new) |
| 96 | + allPackageNames = set(packageNames) |
| 97 | + |
| 98 | + packageFilenames = [] |
| 99 | + while len(packageNames) > 0: |
| 100 | + packName = packageNames.pop() |
| 101 | + package = _findPackage(packName, srcpkg) |
| 102 | + if package == None: |
| 103 | + error('Package %s not found', packName) |
| 104 | + continue |
| 105 | + dependencies = _checkPackageRequirements(package, allPackageNames) |
| 106 | + if withDependencies and len(dependencies) > 0: |
| 107 | + packageNames.extend(dependencies) |
| 108 | + allPackageNames |= dependencies |
| 109 | + localFilenameFull = os.path.join(_packageCacheDirectory, package['filename']) |
| 110 | + if not os.path.exists(localFilenameFull): |
| 111 | + warning('Downloading %s', package['filename']) |
| 112 | + urlretrieve(package['url'], localFilenameFull) |
| 113 | + packageFilenames.append(package['filename']) |
| 114 | + return packageFilenames |
| 115 | + |
| 116 | + |
| 117 | +def _extractFile(filename, output_dir=_extractedCacheDirectory): |
| 118 | + from subprocess import check_call |
| 119 | + try: |
| 120 | + with open('7z.log', 'w') as logfile: |
| 121 | + check_call(['7z', 'x', '-o'+output_dir, '-y', filename], stdout=logfile) |
| 122 | + os.remove('7z.log') |
| 123 | + except: |
| 124 | + error('Failed to extract %s into %s' % (filename, output_dir) ) |
| 125 | + |
| 126 | + |
| 127 | +def packagesExtract(packageFilenames, srcpkg=False): |
| 128 | + for packageFilename in packageFilenames : |
| 129 | + warning('Extracting %s', packageFilename) |
| 130 | + cpioFilename = os.path.join(_extractedCacheDirectory, os.path.splitext(packageFilename)[0] + '.cpio') |
| 131 | + if not os.path.exists(cpioFilename): |
| 132 | + _extractFile(os.path.join(_packageCacheDirectory, packageFilename)) |
| 133 | + if srcpkg: |
| 134 | + _extractFile(cpioFilename, os.path.join(_extractedFilesDirectory, os.path.splitext(packageFilename)[0])) |
| 135 | + else: |
| 136 | + _extractFile(cpioFilename, _extractedFilesDirectory) |
| 137 | + |
| 138 | + |
| 139 | +def GetBaseDirectory(): |
| 140 | + if os.path.exists(os.path.join(_extractedFilesDirectory, 'usr/i686-w64-mingw32/sys-root/mingw')): |
| 141 | + return os.path.join(_extractedFilesDirectory, 'usr/i686-w64-mingw32/sys-root/mingw') |
| 142 | + if os.path.exists(os.path.join(_extractedFilesDirectory, 'usr/x86_64-w64-mingw32/sys-root/mingw')): |
| 143 | + return os.path.join(_extractedFilesDirectory, 'usr/x86_64-w64-mingw32/sys-root/mingw') |
| 144 | + return _extractedFilesDirectory |
| 145 | + |
| 146 | + |
| 147 | +def CleanExtracted(): |
| 148 | + from shutil import rmtree |
| 149 | + rmtree(os.path.join(_extractedFilesDirectory, 'usr'), True) |
| 150 | + |
| 151 | + |
| 152 | +def SetExecutableBit(): |
| 153 | + # set executable bit on libraries and executables |
| 154 | + for root, dirs, files in os.walk(GetBaseDirectory()): |
| 155 | + for filename in {f for f in files if f.endswith('.dll') or f.endswith('.exe')} | set(dirs): |
| 156 | + os.chmod(os.path.join(root, filename), 0o755) |
| 157 | + |
| 158 | + |
| 159 | +def GetOptions(): |
| 160 | + from optparse import OptionParser, OptionGroup #3.2: use argparse |
| 161 | + |
| 162 | + parser = OptionParser(usage="usage: %prog [options] packages", |
| 163 | + description="Easy download of RPM packages for Windows.") |
| 164 | + |
| 165 | + # Options specifiying download repository |
| 166 | + default_project = "windows:mingw:win32" |
| 167 | + default_repository = "openSUSE_11.4" |
| 168 | + default_repo_url = "http://download.opensuse.org/repositories/PROJECT/REPOSITORY/" |
| 169 | + repoOptions = OptionGroup(parser, "Specify download repository") |
| 170 | + repoOptions.add_option("-p", "--project", dest="project", default=default_project, |
| 171 | + metavar="PROJECT", help="Download from PROJECT [%default]") |
| 172 | + repoOptions.add_option("-r", "--repository", dest="repository", default=default_repository, |
| 173 | + metavar="REPOSITORY", help="Download from REPOSITORY [%default]") |
| 174 | + repoOptions.add_option("-u", "--repo-url", dest="repo_url", default=default_repo_url, |
| 175 | + metavar="URL", help="Download packages from URL (overrides PROJECT and REPOSITORY options) [%default]") |
| 176 | + parser.add_option_group(repoOptions) |
| 177 | + |
| 178 | + # Package selection options |
| 179 | + parser.set_defaults(withdeps=False) |
| 180 | + packageOptions = OptionGroup(parser, "Package selection") |
| 181 | + packageOptions.add_option("--deps", action="store_true", dest="withdeps", help="Download dependencies") |
| 182 | + packageOptions.add_option("--no-deps", action="store_false", dest="withdeps", help="Do not download dependencies [default]") |
| 183 | + packageOptions.add_option("--src", action="store_true", dest="srcpkg", default=False, help="Download source instead of noarch package") |
| 184 | + parser.add_option_group(packageOptions) |
| 185 | + |
| 186 | + # Output options |
| 187 | + outputOptions = OptionGroup(parser, "Output options", "Normally the downloaded packages are extracted in the current directory.") |
| 188 | + outputOptions.add_option("--no-clean", action="store_false", dest="clean", default=True, |
| 189 | + help="Do not remove previously extracted files") |
| 190 | + outputOptions.add_option("-z", "--make-zip", action="store_true", dest="makezip", default=False, |
| 191 | + help="Make a zip file of the extracted packages (the name of the zip file is based on the first package specified)") |
| 192 | + outputOptions.add_option("-m", "--add-metadata", action="store_true", dest="metadata", default=False, |
| 193 | + help="Add a file containing package dependencies and provides") |
| 194 | + parser.add_option_group(outputOptions) |
| 195 | + |
| 196 | + # Other options |
| 197 | + parser.add_option("-q", "--quiet", action="store_false", dest="verbose", default=True, |
| 198 | + help="Don't print status messages to stderr") |
| 199 | + |
| 200 | + (options, args) = parser.parse_args() |
| 201 | + |
| 202 | + if len(args) == 0: |
| 203 | + parser.print_help(file=sys.stderr) |
| 204 | + sys.exit(1) |
| 205 | + |
| 206 | + return (options, args) |
| 207 | + |
| 208 | + |
| 209 | +def main(): |
| 210 | + import re, zipfile |
| 211 | + |
| 212 | + (options, args) = GetOptions() |
| 213 | + packages = set(args) |
| 214 | + logging.basicConfig(level=(logging.WARNING if options.verbose else logging.ERROR), format='%(message)s', stream=sys.stderr) |
| 215 | + |
| 216 | + # Open repository |
| 217 | + repository = options.repo_url.replace("PROJECT", options.project.replace(':', ':/')).replace("REPOSITORY", options.repository) |
| 218 | + try: |
| 219 | + OpenRepository(repository) |
| 220 | + except Exception as e: |
| 221 | + sys.exit('Error opening repository:\n\t%s\n\t%s' % (repository, e)) |
| 222 | + |
| 223 | + if options.clean: |
| 224 | + CleanExtracted() |
| 225 | + |
| 226 | + if options.makezip or options.metadata: |
| 227 | + package = _findPackage(args[0]) #if args[0].endswith('.rpm') |
| 228 | + if package == None: package = _findPackage("mingw32-"+args[0], options.srcpkg) |
| 229 | + if package == None: package = _findPackage("mingw64-"+args[0], options.srcpkg) |
| 230 | + if package == None: |
| 231 | + sys.exit('Package not found:\n\t%s' % args[0]) |
| 232 | + packageBasename = re.sub('^mingw(32|64)-|\\.noarch|\\.rpm$', '', package['filename']) |
| 233 | + |
| 234 | + packages = packagesDownload(packages, options.withdeps, options.srcpkg) |
| 235 | + for package in sorted(packages): |
| 236 | + print(package) |
| 237 | + |
| 238 | + packagesExtract(packages, options.srcpkg) |
| 239 | + SetExecutableBit() |
| 240 | + |
| 241 | + if options.metadata: |
| 242 | + cleanup = lambda n: re.sub('^mingw(?:32|64)-(.*)', '\\1', re.sub('^mingw(?:32|64)[(](.*)[)]', '\\1', n)) |
| 243 | + with open(os.path.join(GetBaseDirectory(), packageBasename + '.metadata'), 'w') as m: |
| 244 | + for packageFilename in sorted(packages): |
| 245 | + package = [p for p in _packages if p['filename'] == packageFilename][0] |
| 246 | + m.writelines(['provides:%s\r\n' % cleanup(p) for p in package['provides']]) |
| 247 | + m.writelines(['requires:%s\r\n' % cleanup(r) for r in package['requires']]) |
| 248 | + |
| 249 | + if options.makezip: |
| 250 | + packagezip = zipfile.ZipFile(packageBasename + '.zip', 'w', compression=zipfile.ZIP_DEFLATED) |
| 251 | + for root, dirs, files in os.walk(GetBaseDirectory()): |
| 252 | + for filename in files: |
| 253 | + fullname = os.path.join(root, filename) |
| 254 | + packagezip.write(fullname, fullname.replace(GetBaseDirectory(), '')) |
| 255 | + packagezip.close() #3.2: use with |
| 256 | + if options.clean: |
| 257 | + CleanExtracted() |
| 258 | + |
| 259 | +if __name__ == "__main__": |
| 260 | + main() |
| 261 | + |
0 commit comments