Merge branch 'develop' into new-changelog-parsing

Author: jonoomph

.gitlab-ci.yml

@@ -22,14 +22,14 @@ linux-builder:
     - git log $(git describe --tags --abbrev=0 @^)..@ --oneline --pretty=format:"%C(auto,yellow)%h%C(auto,magenta)% %C(auto,blue)%>(12,trunc)%ad %C(auto,green)%<(25,trunc)%aN%C(auto,reset)%s%C(auto,red)% gD% D" --date=short > "build/install-x64/share/$CI_PROJECT_NAME.log"
     - cd doc; make html SPHINXOPTS="-D html_theme_options.analytics_id=UA-4381101-5"; cd ..;
     - ~/auto-update-sphinx "$CI_PROJECT_DIR/build" "$CI_COMMIT_REF_NAME"
-    - python3 freeze.py build
-    - /bin/sh ./installer/mangle-hw-libs.sh "build/exe.linux-x86_64-3.4/"
-    - python3 installer/build-server.py "$SLACK_TOKEN" "$S3_ACCESS_KEY" "$S3_SECRET_KEY" "$WINDOWS_KEY" "$WINDOWS_PASSWORD" "$GITHUB_USER" "$GITHUB_PASS" "False" "$CI_COMMIT_REF_NAME"
+    - python3 -u freeze.py build
+    - for dir in "build/*/"; do /bin/sh ./installer/mangle-hw-libs.sh $(realpath "${dir}"); done 
+    - python3 -u installer/build-server.py "$SLACK_TOKEN" "$S3_ACCESS_KEY" "$S3_SECRET_KEY" "$WINDOWS_KEY" "$WINDOWS_PASSWORD" "$GITHUB_USER" "$GITHUB_PASS" "False" "$CI_COMMIT_REF_NAME"
   when: always
   except:
   - tags
   tags:
-    - linux
+    - linux-bionic
 
 mac-builder:
   stage: build-openshot-qt
@@ -63,8 +63,8 @@ mac-builder:
     - export DYLD_LIBRARY_PATH=$CI_PROJECT_DIR/build/install-x64/lib:DYLD_LIBRARY_PATH
     - echo -e "CI_PROJECT_NAME:$CI_PROJECT_NAME\nCI_COMMIT_REF_NAME:$CI_COMMIT_REF_NAME\nCI_COMMIT_SHA:$CI_COMMIT_SHA\nCI_JOB_ID:$CI_JOB_ID" > "build/install-x64/share/$CI_PROJECT_NAME"
     - git log $(git describe --tags --abbrev=0 @^)..@ --oneline --pretty=format:"%C(auto,yellow)%h%C(auto,magenta)% %C(auto,blue)%>(12,trunc)%ad %C(auto,green)%<(25,trunc)%aN%C(auto,reset)%s%C(auto,red)% gD% D" --date=short > "build/install-x64/share/$CI_PROJECT_NAME.log"
-    - /Library/Frameworks/Python.Framework/Versions/3.6/bin/python3.6 freeze.py bdist_mac --iconfile=installer/openshot.icns --custom-info-plist=installer/Info.plist --bundle-name="OpenShot Video Editor"
-    - /Library/Frameworks/Python.Framework/Versions/3.6/bin/python3.6 installer/build-server.py "$SLACK_TOKEN" "$S3_ACCESS_KEY" "$S3_SECRET_KEY" "$WINDOWS_KEY" "$WINDOWS_PASSWORD" "$GITHUB_USER" "$GITHUB_PASS" "False" "$CI_COMMIT_REF_NAME"
+    - /Library/Frameworks/Python.Framework/Versions/3.6/bin/python3.6 -u freeze.py bdist_mac --iconfile=installer/openshot.icns --custom-info-plist=installer/Info.plist --bundle-name="OpenShot Video Editor"
+    - /Library/Frameworks/Python.Framework/Versions/3.6/bin/python3.6 -u installer/build-server.py "$SLACK_TOKEN" "$S3_ACCESS_KEY" "$S3_SECRET_KEY" "$WINDOWS_KEY" "$WINDOWS_PASSWORD" "$GITHUB_USER" "$GITHUB_PASS" "False" "$CI_COMMIT_REF_NAME"
   when: always
   except:
   - tags
@@ -90,8 +90,8 @@ windows-builder-x64:
     - New-Item -path "build/install-x64/share/" -Name "$CI_PROJECT_NAME" -Value "CI_PROJECT_NAME:$CI_PROJECT_NAME`nCI_COMMIT_REF_NAME:$CI_COMMIT_REF_NAME`nCI_COMMIT_SHA:$CI_COMMIT_SHA`nCI_JOB_ID:$CI_JOB_ID" -ItemType file -force
     - $PREV_GIT_LABEL=(git describe --tags --abbrev=0 '@^')
     - git log "$PREV_GIT_LABEL..@" --oneline --pretty=format:"%C(auto,yellow)%h%C(auto,magenta)% %C(auto,blue)%>(12,trunc)%ad %C(auto,green)%<(25,trunc)%aN%C(auto,reset)%s%C(auto,red)% gD% D" --date=short > "build/install-x64/share/$CI_PROJECT_NAME.log"
-    - python3 freeze.py build
-    - python3 installer/build-server.py "$SLACK_TOKEN" "$S3_ACCESS_KEY" "$S3_SECRET_KEY" "$WINDOWS_KEY" "$WINDOWS_PASSWORD" "$GITHUB_USER" "$GITHUB_PASS" "False" "$CI_COMMIT_REF_NAME"
+    - python3 -u freeze.py build
+    - python3 -u installer/build-server.py "$SLACK_TOKEN" "$S3_ACCESS_KEY" "$S3_SECRET_KEY" "$WINDOWS_KEY" "$WINDOWS_PASSWORD" "$GITHUB_USER" "$GITHUB_PASS" "False" "$CI_COMMIT_REF_NAME"
   when: always
   except:
   - tags
@@ -117,9 +117,9 @@ windows-builder-x86:
     - New-Item -path "build/install-x86/share/" -Name "$CI_PROJECT_NAME" -Value "CI_PROJECT_NAME:$CI_PROJECT_NAME`nCI_COMMIT_REF_NAME:$CI_COMMIT_REF_NAME`nCI_COMMIT_SHA:$CI_COMMIT_SHA`nCI_JOB_ID:$CI_JOB_ID" -ItemType file -force
     - $PREV_GIT_LABEL=(git describe --tags --abbrev=0 '@^')
     - git log "$PREV_GIT_LABEL..@" --oneline --pretty=format:"%C(auto,yellow)%h%C(auto,magenta)% %C(auto,blue)%>(12,trunc)%ad %C(auto,green)%<(25,trunc)%aN%C(auto,reset)%s%C(auto,red)% gD% D" --date=short > "build/install-x86/share/$CI_PROJECT_NAME.log"
-    - python3 freeze.py build
+    - python3 -u freeze.py build
     - editbin /LARGEADDRESSAWARE "$CI_PROJECT_DIR\build\exe.mingw-3.7\openshot-qt.exe"
-    - python3 installer/build-server.py "$SLACK_TOKEN" "$S3_ACCESS_KEY" "$S3_SECRET_KEY" "$WINDOWS_KEY" "$WINDOWS_PASSWORD" "$GITHUB_USER" "$GITHUB_PASS" "True" "$CI_COMMIT_REF_NAME"
+    - python3 -u installer/build-server.py "$SLACK_TOKEN" "$S3_ACCESS_KEY" "$S3_SECRET_KEY" "$WINDOWS_KEY" "$WINDOWS_PASSWORD" "$GITHUB_USER" "$GITHUB_PASS" "True" "$CI_COMMIT_REF_NAME"
   when: always
   except:
   - tags

.travis.yml

@@ -7,38 +7,58 @@ addons:
   apt:
     sources:
     - sourceline: 'ppa:openshot.developers/libopenshot-daily'
-    packages:
+    packages: &p_common
     - libopenshot-audio-dev
     - libopenshot-dev
     - libssl-dev
     - python3-openshot
     - python3-pip
     - python3-setuptools
     - python3-pyqt5
-    - python3-pyqt5.qtmultimedia
-    - python3-pyqt5.qtopengl
     - python3-pyqt5.qtsvg
-    - python3-pyqt5.qtwebkit
     - python3-requests
     - python3-xdg
     - python3-zmq
     - qttranslations5-l10n
     - xvfb
 
-matrix:
+jobs:
   include:
     - name: "Python 3.5 (Ubuntu 16.04 Xenial)"
       os: linux
       dist: xenial
       python: 3.5
+      addons:
+        apt:
+          sources:
+          - sourceline: 'ppa:openshot.developers/libopenshot-daily'
+          packages:
+          - *p_common
+          - python3-pyqt5.qtwebkit
     - name: "Python 3.6 (Ubuntu 18.04 Bionic)"
       os: linux
       dist: bionic
       python: 3.6
+      addons:
+        apt:
+          sources:
+          - sourceline: 'ppa:openshot.developers/libopenshot-daily'
+          packages:
+          - *p_common
+          - python3-pyqt5.qtwebengine
+          - python3-pyqt5.qtopengl
     - name: "Python 3.8 (Ubuntu 20.04 Focal)"
       os: linux
       dist: focal
       python: 3.8
+      addons:
+        apt:
+          sources:
+          - sourceline: 'ppa:openshot.developers/libopenshot-daily'
+          packages:
+          - *p_common
+          - python3-pyqt5.qtwebengine
+          - python3-pyqt5.qtopengl
 
 install:
   - pip3 install 'cx-freeze==6.1'

COPYING

@@ -622,17 +622,18 @@ copy of the Program in return for a fee.
   18. Exceptions
 
   The OpenShot Video Editor project hereby grants permission for non-GPL 
-compatible GStreamer, FFMPEG, and libopenshot to be used and distributed together with 
-GStreamer, FFMPEG, MLT, and OpenShot Video Editor. This permission is above and beyond 
-the permissions granted by the GPL license by which OpenShot Video Editor 
-is covered. If you modify this code, you may extend this exception to 
-your version of the code, but you are not obligated to do so. If you do 
-not wish to do so, delete this exception statement from your version.
-
-  OpenShot Video Editor does not contain or use any proprietary codecs.  We support 
-free and open-source codecs, such as Ogg Vorbis and Theora. However, since we use 
-the ffmpeg library, it is possible to use any ffmpeg supported codec, assuming
-you have legal permission to do so.
+compatible GStreamer, FFMPEG, and libopenshot to be used and distributed
+together with GStreamer, FFMPEG, MLT, and OpenShot Video Editor. This
+permission is above and beyond the permissions granted by the GPL license
+by which OpenShot Video Editor is covered. If you modify this code, you
+may extend this exception to your version of the code, but you are not
+obligated to do so. If you do not wish to do so, delete this exception
+statement from your version.
+
+  OpenShot Video Editor does not contain or use any proprietary codecs.
+We support free and open-source codecs, such as Ogg Vorbis and Theora.
+However, since we use the ffmpeg library, it is possible to use any
+ffmpeg supported codec, assuming you have legal permission to do so.
 
 
-                     END OF TERMS AND CONDITIONS
+                     END OF TERMS AND CONDITIONS

doc/titles.rst

@@ -76,6 +76,7 @@
                    "openshot",
                    "time",
                    "uuid",
+                   "idna",
                    "shutil",
                    "threading",
                    "subprocess",
@@ -90,6 +91,9 @@
                    "json"
                    ]
 
+# Modules to include
+python_modules = ["idna.idnadata"]
+
 # Determine absolute PATH of OpenShot folder
 PATH = os.path.dirname(os.path.realpath(__file__))  # Primary openshot folder
 
@@ -210,6 +214,29 @@ def find_files(directory, patterns):
     if os.path.exists(resvg_path):
         external_so_files.append((resvg_path, os.path.basename(resvg_path)))
 
+    # Add QtWebEngineProcess (if found)
+    web_process_path = ARCHLIB + "qt5/libexec/QtWebEngineProcess"
+    if os.path.exists(web_process_path):
+        external_so_files.append(
+            (web_process_path, os.path.basename(web_process_path)))
+
+    # Add QtWebEngineProcess Resources & Local
+    qt5_path = "/usr/share/qt5/"
+    for filename in find_files(os.path.join(qt5_path, "resources"), ["*"]):
+        external_so_files.append((filename, os.path.relpath(filename, start=qt5_path)))
+    for filename in find_files(os.path.join(qt5_path, "translations", "qtwebengine_locales"), ["*"]):
+        external_so_files.append((filename, os.path.relpath(filename, start=qt5_path)))
+
+    # Add Qt xcbglintegrations plugin
+    xcbgl_path = ARCHLIB + "qt5/"
+    for filename in find_files(os.path.join(xcbgl_path, "plugins", "xcbglintegrations"), ["*"]):
+        external_so_files.append((filename, os.path.relpath(filename, start=xcbgl_path)))
+
+    # Add libsoftokn3
+    nss_path = ARCHLIB + "nss/"
+    for filename in find_files(nss_path, ["*"]):
+        external_so_files.append((filename, os.path.basename(filename)))
+
     # Append Linux ICON file
     iconFile += ".svg"
     src_files.append((os.path.join(PATH, "xdg", iconFile), iconFile))
@@ -223,15 +250,32 @@ def find_files(directory, patterns):
     # Get a list of all openshot.so dependencies (scan these libraries for their dependencies)
     pyqt5_mod_files = []
     from importlib import import_module
-    for submod in ['Qt', 'QtWebKit', 'QtSvg', 'QtWebKitWidgets', 'QtWidgets', 'QtCore', 'QtGui', 'QtDBus']:
+    for submod in ['Qt', 'QtSvg', 'QtWidgets', 'QtCore', 'QtGui', 'QtDBus']:
         mod_name  = "PyQt5.{}".format(submod)
         mod = import_module(mod_name)
         pyqt5_mod_files.append(inspect.getfile(mod))
-
-    lib_list = [os.path.join(libopenshot_path, "libopenshot.so"),
-                "/usr/local/lib/libresvg.so",
-                ARCHLIB + "qt5/plugins/platforms/libqxcb.so"
-                ] + pyqt5_mod_files
+    # Optional additions
+    for mod_name in [
+            'PyQt5.QtWebEngine',
+            'PyQt5.QtWebEngineWidgets',
+            'PyQt5.QtWebKit',
+            'PyQt5.QtWebKitWidgets',
+            ]:
+        try:
+            mod = import_module(mod_name)
+            pyqt5_mod_files.append(inspect.getfile(mod))
+        except ImportError as ex:
+            log.warning("Skipping {}: {}".format(mod_name, ex))
+
+
+    lib_list = pyqt5_mod_files
+    for lib_name in [
+            os.path.join(libopenshot_path, "libopenshot.so"),
+            "/usr/local/lib/libresvg.so",
+            ARCHLIB + "qt5/plugins/platforms/libqxcb.so"
+            ]:
+        if os.path.exists(lib_name):
+            lib_list.append(lib_name)
 
     import subprocess
     for library in lib_list:
@@ -259,6 +303,7 @@ def find_files(directory, patterns):
             libpath = libdetailsparts[0].strip()
             libpath_file = os.path.basename(libpath)
             if (libpath
+                and os.path.exists(libpath)
                 and not libpath.startswith("/lib")
                 and "libnvidia-glcore.so" not in libpath
                 and libpath_file not in [
@@ -282,31 +327,20 @@ def find_files(directory, patterns):
                     "libpangoft2-1.0.so.0",
                     "libharfbuzz.so.0",
                     "libthai.so.0",
-                ]
-                and not libpath_file.startswith("libxcb-")
-                ) \
-               or libpath_file in ["libgcrypt.so.11", "libQt5DBus.so.5", "libpng12.so.0", "libbz2.so.1.0", "libqxcb.so"]:
-
-                # Ignore missing files
-                if os.path.exists(libpath):
-                    filepath, filename = os.path.split(libpath)
-                    external_so_files.append((libpath, filename))
-
-    # Manually add missing files (that were missed in the above step). These files are required
-    # for certain distros (like Fedora, openSUSE, Debian, etc...)
-    # Also add Glib related files (required for some distros)
-
-    for added_lib in [ARCHLIB + "libssl.so",
-                      ARCHLIB + "libcrypto.so",
-                      ARCHLIB + "libglib-2.0.so",
-                      ARCHLIB + "libgio-2.0.so",
-                      ARCHLIB + "libgmodule-2.0.so",
-                      ARCHLIB + "libthread-2.0.so"
-                      ]:
-        if os.path.exists(added_lib):
-            external_so_files.append((added_lib, os.path.basename(added_lib)))
-        else:
-            log.warning("{}: not found, skipping".format(added_lib))
+                    ]
+               ) or libpath_file in [
+                    "libgcrypt.so.11",
+                    "libQt5DBus.so.5",
+                    "libpng12.so.0",
+                    "libbz2.so.1.0",
+                    "libqxcb.so",
+                    "libxcb-xinerama.so.0",
+                    "libpcre.so.3",
+                    "libselinux.so.1",
+                    ]:
+                external_so_files.append((libpath, libpath_file))
+                # Any other lib deps that fail to meet the inclusion
+                # criteria above will be silently skipped over
 
 elif sys.platform == "darwin":
     # Copy Mac specific files that cx_Freeze misses
@@ -327,6 +361,25 @@ def find_files(directory, patterns):
     iconFile += ".hqx"
     src_files.append((os.path.join(PATH, "xdg", iconFile), iconFile))
 
+    # Add QtWebEngineProcess (if found)
+    qt_install_path = "/usr/local/qt5.15.X/qt5.15/5.15.0/clang_64/"
+    qt_webengine_path = os.path.join(qt_install_path, "lib", "QtWebEngineCore.framework", "Versions", "5")
+    web_process_path = os.path.join(qt_webengine_path, "Helpers", "QtWebEngineProcess.app", "Contents", "MacOS", "QtWebEngineProcess")
+    web_core_path = os.path.join(qt_webengine_path, "QtWebEngineCore")
+    external_so_files.append((web_process_path, os.path.basename(web_process_path)))
+    external_so_files.append((web_core_path, os.path.basename(web_core_path)))
+
+    # Add QtWebEngineProcess Resources & Local
+    for filename in find_files(os.path.join(qt_webengine_path, "Resources"), ["*"]):
+        external_so_files.append((filename, os.path.relpath(filename, start=os.path.join(qt_webengine_path, "Resources"))))
+    for filename in find_files(os.path.join(qt_webengine_path, "Resources", "qtwebengine_locales"), ["*"]):
+        external_so_files.append((filename, os.path.relpath(filename, start=os.path.join(qt_webengine_path, "Resources"))))
+    for filename in find_files(os.path.join(qt_install_path, "plugins"), ["*"]):
+        relative_filepath = os.path.relpath(filename, start=os.path.join(qt_install_path, "plugins"))
+        plugin_name = os.path.dirname(relative_filepath)
+        if plugin_name not in ["sqldrivers", "playlistformats", "gamepads", "bearer"]:
+            external_so_files.append((filename, relative_filepath))
+
 # Append all source files
 src_files.append((os.path.join(PATH, "installer", "qt.conf"), "qt.conf"))
 for filename in find_files("openshot_qt", ["*"]):
@@ -335,6 +388,7 @@ def find_files(directory, patterns):
 # Dependencies are automatically detected, but it might need fine tuning.
 build_exe_options["packages"] = python_packages
 build_exe_options["include_files"] = src_files + external_so_files
+build_exe_options["includes"] = python_modules
 
 # Set options
 build_options["build_exe"] = build_exe_options