Merge pull request #3604 from OpenShot/new-webengine-support

Initial support for QtWebEngine and Async callbacks for timeline/Qt Integration

Author: jonoomph

.gitlab-ci.yml

@@ -22,14 +22,14 @@ linux-builder:
     - git log $(git describe --tags --abbrev=0 @^)..@ --oneline --pretty=format:"%C(auto,yellow)%h%C(auto,magenta)% %C(auto,blue)%>(12,trunc)%ad %C(auto,green)%<(25,trunc)%aN%C(auto,reset)%s%C(auto,red)% gD% D" --date=short > "build/install-x64/share/$CI_PROJECT_NAME.log"
     - cd doc; make html SPHINXOPTS="-D html_theme_options.analytics_id=UA-4381101-5"; cd ..;
     - ~/auto-update-sphinx "$CI_PROJECT_DIR/build" "$CI_COMMIT_REF_NAME"
-    - python3 freeze.py build
+    - python3 -u freeze.py build
     - /bin/sh ./installer/mangle-hw-libs.sh "build/exe.linux-x86_64-3.4/"
-    - python3 installer/build-server.py "$SLACK_TOKEN" "$S3_ACCESS_KEY" "$S3_SECRET_KEY" "$WINDOWS_KEY" "$WINDOWS_PASSWORD" "$GITHUB_USER" "$GITHUB_PASS" "False" "$CI_COMMIT_REF_NAME"
+    - python3 -u installer/build-server.py "$SLACK_TOKEN" "$S3_ACCESS_KEY" "$S3_SECRET_KEY" "$WINDOWS_KEY" "$WINDOWS_PASSWORD" "$GITHUB_USER" "$GITHUB_PASS" "False" "$CI_COMMIT_REF_NAME"
   when: always
   except:
   - tags
   tags:
-    - linux
+    - linux-bionic
 
 mac-builder:
   stage: build-openshot-qt
@@ -63,8 +63,8 @@ mac-builder:
     - export DYLD_LIBRARY_PATH=$CI_PROJECT_DIR/build/install-x64/lib:DYLD_LIBRARY_PATH
     - echo -e "CI_PROJECT_NAME:$CI_PROJECT_NAME\nCI_COMMIT_REF_NAME:$CI_COMMIT_REF_NAME\nCI_COMMIT_SHA:$CI_COMMIT_SHA\nCI_JOB_ID:$CI_JOB_ID" > "build/install-x64/share/$CI_PROJECT_NAME"
     - git log $(git describe --tags --abbrev=0 @^)..@ --oneline --pretty=format:"%C(auto,yellow)%h%C(auto,magenta)% %C(auto,blue)%>(12,trunc)%ad %C(auto,green)%<(25,trunc)%aN%C(auto,reset)%s%C(auto,red)% gD% D" --date=short > "build/install-x64/share/$CI_PROJECT_NAME.log"
-    - /Library/Frameworks/Python.Framework/Versions/3.6/bin/python3.6 freeze.py bdist_mac --iconfile=installer/openshot.icns --custom-info-plist=installer/Info.plist --bundle-name="OpenShot Video Editor"
-    - /Library/Frameworks/Python.Framework/Versions/3.6/bin/python3.6 installer/build-server.py "$SLACK_TOKEN" "$S3_ACCESS_KEY" "$S3_SECRET_KEY" "$WINDOWS_KEY" "$WINDOWS_PASSWORD" "$GITHUB_USER" "$GITHUB_PASS" "False" "$CI_COMMIT_REF_NAME"
+    - /Library/Frameworks/Python.Framework/Versions/3.6/bin/python3.6 -u freeze.py bdist_mac --iconfile=installer/openshot.icns --custom-info-plist=installer/Info.plist --bundle-name="OpenShot Video Editor"
+    - /Library/Frameworks/Python.Framework/Versions/3.6/bin/python3.6 -u installer/build-server.py "$SLACK_TOKEN" "$S3_ACCESS_KEY" "$S3_SECRET_KEY" "$WINDOWS_KEY" "$WINDOWS_PASSWORD" "$GITHUB_USER" "$GITHUB_PASS" "False" "$CI_COMMIT_REF_NAME"
   when: always
   except:
   - tags
@@ -90,8 +90,8 @@ windows-builder-x64:
     - New-Item -path "build/install-x64/share/" -Name "$CI_PROJECT_NAME" -Value "CI_PROJECT_NAME:$CI_PROJECT_NAME`nCI_COMMIT_REF_NAME:$CI_COMMIT_REF_NAME`nCI_COMMIT_SHA:$CI_COMMIT_SHA`nCI_JOB_ID:$CI_JOB_ID" -ItemType file -force
     - $PREV_GIT_LABEL=(git describe --tags --abbrev=0 '@^')
     - git log "$PREV_GIT_LABEL..@" --oneline --pretty=format:"%C(auto,yellow)%h%C(auto,magenta)% %C(auto,blue)%>(12,trunc)%ad %C(auto,green)%<(25,trunc)%aN%C(auto,reset)%s%C(auto,red)% gD% D" --date=short > "build/install-x64/share/$CI_PROJECT_NAME.log"
-    - python3 freeze.py build
-    - python3 installer/build-server.py "$SLACK_TOKEN" "$S3_ACCESS_KEY" "$S3_SECRET_KEY" "$WINDOWS_KEY" "$WINDOWS_PASSWORD" "$GITHUB_USER" "$GITHUB_PASS" "False" "$CI_COMMIT_REF_NAME"
+    - python3 -u freeze.py build
+    - python3 -u installer/build-server.py "$SLACK_TOKEN" "$S3_ACCESS_KEY" "$S3_SECRET_KEY" "$WINDOWS_KEY" "$WINDOWS_PASSWORD" "$GITHUB_USER" "$GITHUB_PASS" "False" "$CI_COMMIT_REF_NAME"
   when: always
   except:
   - tags
@@ -117,9 +117,9 @@ windows-builder-x86:
     - New-Item -path "build/install-x86/share/" -Name "$CI_PROJECT_NAME" -Value "CI_PROJECT_NAME:$CI_PROJECT_NAME`nCI_COMMIT_REF_NAME:$CI_COMMIT_REF_NAME`nCI_COMMIT_SHA:$CI_COMMIT_SHA`nCI_JOB_ID:$CI_JOB_ID" -ItemType file -force
     - $PREV_GIT_LABEL=(git describe --tags --abbrev=0 '@^')
     - git log "$PREV_GIT_LABEL..@" --oneline --pretty=format:"%C(auto,yellow)%h%C(auto,magenta)% %C(auto,blue)%>(12,trunc)%ad %C(auto,green)%<(25,trunc)%aN%C(auto,reset)%s%C(auto,red)% gD% D" --date=short > "build/install-x86/share/$CI_PROJECT_NAME.log"
-    - python3 freeze.py build
+    - python3 -u freeze.py build
     - editbin /LARGEADDRESSAWARE "$CI_PROJECT_DIR\build\exe.mingw-3.7\openshot-qt.exe"
-    - python3 installer/build-server.py "$SLACK_TOKEN" "$S3_ACCESS_KEY" "$S3_SECRET_KEY" "$WINDOWS_KEY" "$WINDOWS_PASSWORD" "$GITHUB_USER" "$GITHUB_PASS" "True" "$CI_COMMIT_REF_NAME"
+    - python3 -u installer/build-server.py "$SLACK_TOKEN" "$S3_ACCESS_KEY" "$S3_SECRET_KEY" "$WINDOWS_KEY" "$WINDOWS_PASSWORD" "$GITHUB_USER" "$GITHUB_PASS" "True" "$CI_COMMIT_REF_NAME"
   when: always
   except:
   - tags

.travis.yml

@@ -18,14 +18,17 @@ addons:
     - python3-pyqt5.qtmultimedia
     - python3-pyqt5.qtopengl
     - python3-pyqt5.qtsvg
-    - python3-pyqt5.qtwebkit
+    - python3-pyqt5.qtwebengine
     - python3-requests
     - python3-xdg
     - python3-zmq
     - qttranslations5-l10n
     - xvfb
 
-matrix:
+jobs:
+  allow_failures:
+    - name: "Python 3.5 (Ubuntu 16.04 Xenial)"
+
   include:
     - name: "Python 3.5 (Ubuntu 16.04 Xenial)"
       os: linux

freeze.py

@@ -76,6 +76,7 @@
                    "openshot",
                    "time",
                    "uuid",
+                   "idna",
                    "shutil",
                    "threading",
                    "subprocess",
@@ -90,6 +91,9 @@
                    "json"
                    ]
 
+# Modules to include
+python_modules = ["idna.idnadata"]
+
 # Determine absolute PATH of OpenShot folder
 PATH = os.path.dirname(os.path.realpath(__file__))  # Primary openshot folder
 
@@ -210,6 +214,27 @@ def find_files(directory, patterns):
     if os.path.exists(resvg_path):
         external_so_files.append((resvg_path, os.path.basename(resvg_path)))
 
+    # Add QtWebEngineProcess (if found)
+    web_process_path = ARCHLIB + "qt5/libexec/QtWebEngineProcess"
+    external_so_files.append((web_process_path, os.path.basename(web_process_path)))
+
+    # Add QtWebEngineProcess Resources & Local
+    qt5_path = "/usr/share/qt5/"
+    for filename in find_files(os.path.join(qt5_path, "resources"), ["*"]):
+        external_so_files.append((filename, os.path.relpath(filename, start=qt5_path)))
+    for filename in find_files(os.path.join(qt5_path, "translations", "qtwebengine_locales"), ["*"]):
+        external_so_files.append((filename, os.path.relpath(filename, start=qt5_path)))
+
+    # Add Qt xcbglintegrations plugin
+    xcbgl_path = ARCHLIB + "qt5/"
+    for filename in find_files(os.path.join(xcbgl_path, "plugins", "xcbglintegrations"), ["*"]):
+        external_so_files.append((filename, os.path.relpath(filename, start=xcbgl_path)))
+
+    # Add libsoftokn3
+    nss_path = ARCHLIB + "nss/"
+    for filename in find_files(nss_path, ["*"]):
+        external_so_files.append((filename, os.path.basename(filename)))
+
     # Append Linux ICON file
     iconFile += ".svg"
     src_files.append((os.path.join(PATH, "xdg", iconFile), iconFile))
@@ -223,7 +248,7 @@ def find_files(directory, patterns):
     # Get a list of all openshot.so dependencies (scan these libraries for their dependencies)
     pyqt5_mod_files = []
     from importlib import import_module
-    for submod in ['Qt', 'QtWebKit', 'QtSvg', 'QtWebKitWidgets', 'QtWidgets', 'QtCore', 'QtGui', 'QtDBus']:
+    for submod in ['Qt', 'QtWebEngine', 'QtWebEngineWidgets', 'QtSvg', 'QtWidgets', 'QtCore', 'QtGui', 'QtDBus']:
         mod_name  = "PyQt5.{}".format(submod)
         mod = import_module(mod_name)
         pyqt5_mod_files.append(inspect.getfile(mod))
@@ -281,33 +306,14 @@ def find_files(directory, patterns):
                     "libpangocairo-1.0.so.0",
                     "libpangoft2-1.0.so.0",
                     "libharfbuzz.so.0",
-                    "libthai.so.0",
-                ]
-                and not libpath_file.startswith("libxcb-")
-                ) \
-               or libpath_file in ["libgcrypt.so.11", "libQt5DBus.so.5", "libpng12.so.0", "libbz2.so.1.0", "libqxcb.so"]:
-
+                    "libthai.so.0" ]
+                ) or libpath_file in ["libgcrypt.so.11", "libQt5DBus.so.5", "libpng12.so.0",
+                                      "libbz2.so.1.0", "libqxcb.so", "libxcb-xinerama.so.0", "libpcre.so.3"]:
                 # Ignore missing files
                 if os.path.exists(libpath):
                     filepath, filename = os.path.split(libpath)
                     external_so_files.append((libpath, filename))
 
-    # Manually add missing files (that were missed in the above step). These files are required
-    # for certain distros (like Fedora, openSUSE, Debian, etc...)
-    # Also add Glib related files (required for some distros)
-
-    for added_lib in [ARCHLIB + "libssl.so",
-                      ARCHLIB + "libcrypto.so",
-                      ARCHLIB + "libglib-2.0.so",
-                      ARCHLIB + "libgio-2.0.so",
-                      ARCHLIB + "libgmodule-2.0.so",
-                      ARCHLIB + "libthread-2.0.so"
-                      ]:
-        if os.path.exists(added_lib):
-            external_so_files.append((added_lib, os.path.basename(added_lib)))
-        else:
-            log.warning("{}: not found, skipping".format(added_lib))
-
 elif sys.platform == "darwin":
     # Copy Mac specific files that cx_Freeze misses
     # JPEG library
@@ -327,6 +333,25 @@ def find_files(directory, patterns):
     iconFile += ".hqx"
     src_files.append((os.path.join(PATH, "xdg", iconFile), iconFile))
 
+    # Add QtWebEngineProcess (if found)
+    qt_install_path = "/usr/local/qt5.15.X/qt5.15/5.15.0/clang_64/"
+    qt_webengine_path = os.path.join(qt_install_path, "lib", "QtWebEngineCore.framework", "Versions", "5")
+    web_process_path = os.path.join(qt_webengine_path, "Helpers", "QtWebEngineProcess.app", "Contents", "MacOS", "QtWebEngineProcess")
+    web_core_path = os.path.join(qt_webengine_path, "QtWebEngineCore")
+    external_so_files.append((web_process_path, os.path.basename(web_process_path)))
+    external_so_files.append((web_core_path, os.path.basename(web_core_path)))
+
+    # Add QtWebEngineProcess Resources & Local
+    for filename in find_files(os.path.join(qt_webengine_path, "Resources"), ["*"]):
+        external_so_files.append((filename, os.path.relpath(filename, start=os.path.join(qt_webengine_path, "Resources"))))
+    for filename in find_files(os.path.join(qt_webengine_path, "Resources", "qtwebengine_locales"), ["*"]):
+        external_so_files.append((filename, os.path.relpath(filename, start=os.path.join(qt_webengine_path, "Resources"))))
+    for filename in find_files(os.path.join(qt_install_path, "plugins"), ["*"]):
+        relative_filepath = os.path.relpath(filename, start=os.path.join(qt_install_path, "plugins"))
+        plugin_name = os.path.dirname(relative_filepath)
+        if plugin_name not in ["sqldrivers", "playlistformats", "gamepads", "bearer"]:
+            external_so_files.append((filename, relative_filepath))
+
 # Append all source files
 src_files.append((os.path.join(PATH, "installer", "qt.conf"), "qt.conf"))
 for filename in find_files("openshot_qt", ["*"]):
@@ -335,6 +360,7 @@ def find_files(directory, patterns):
 # Dependencies are automatically detected, but it might need fine tuning.
 build_exe_options["packages"] = python_packages
 build_exe_options["include_files"] = src_files + external_so_files
+build_exe_options["includes"] = python_modules
 
 # Set options
 build_options["build_exe"] = build_exe_options