Implement HTML bookmark import job for QtWebEngine

Author: fnkkio

src/modules/backends/web/qtwebengine/QtWebEngineWebBackend.cpp

@@ -23,6 +23,7 @@
 #include "QtWebEngineTransfer.h"
 #include "QtWebEngineUrlRequestInterceptor.h"
 #include "QtWebEngineWebWidget.h"
+#include "../../../../core/BookmarksManager.h"
 #include "../../../../core/ContentFiltersManager.h"
 #include "../../../../core/HandlersManager.h"
 #include "../../../../core/NetworkManagerFactory.h"
@@ -276,6 +277,11 @@ WebWidget* QtWebEngineWebBackend::createWidget(const QVariantMap &parameters, Co
 	return new QtWebEngineWebWidget(parameters, this, parent);
 }
 
+BookmarksImportJob *QtWebEngineWebBackend::createBookmarksImportJob(BookmarksModel::Bookmark *folder, const QString &path, bool areDuplicatesAllowed)
+{
+	return new QtWebEngineBookmarksImportJob(folder, path, areDuplicatesAllowed, this);
+}
+
 QString QtWebEngineWebBackend::getName() const
 {
 	return QLatin1String("qtwebengine");
@@ -344,4 +350,181 @@ WebBackend::BackendCapabilities QtWebEngineWebBackend::getCapabilities() const
 	return (UserScriptsCapability | GlobalCookiesPolicyCapability | GlobalContentFilteringCapability | GlobalDoNotTrackCapability | GlobalProxyCapability | GlobalReferrerCapability | GlobalUserAgentCapability);
 }
 
+QtWebEngineBookmarksImportJob::QtWebEngineBookmarksImportJob(BookmarksModel::Bookmark *folder, const QString &path, bool areDuplicatesAllowed, QObject *parent) : BookmarksImportJob(folder, areDuplicatesAllowed, parent),
+	m_path(path),
+	m_currentAmount(0),
+	m_totalAmount(-1),
+	m_isRunning(false)
+{
+}
+
+void QtWebEngineBookmarksImportJob::start()
+{
+	QFile file(m_path);
+
+	if (!file.open(QIODevice::ReadOnly))
+	{
+		emit importFinished(Importer::BookmarksImport, Importer::FailedImport, 0);
+		emit jobFinished(false);
+
+		deleteLater();
+
+		return;
+	}
+
+	m_isRunning = true;
+
+	QEventLoop loop;
+	QWebEnginePage page;
+
+	connect(&page, &QWebEnginePage::loadFinished,
+			[this, &page, &loop](bool ok) {
+
+		if (!ok)
+		{
+			m_totalAmount = 0;
+			loop.exit();
+			return;
+		}
+
+		QVariant jsResult;
+
+		jsResult = execJavascript(page, "document.getElementsByTagName('DT').length");
+		m_totalAmount = jsResult.toInt(&ok);
+		if (!ok)
+		{
+			m_totalAmount = 0;
+			loop.exit();
+			return;
+		}
+
+		emit importStarted(Importer::BookmarksImport, m_totalAmount);
+
+		jsResult = execJavascript(page, "document.querySelectorAll('a,hr').length");
+		const int numLinks = jsResult.toInt(&ok);
+
+		BookmarksManager::getModel()->beginImport(getImportFolder(), numLinks, numLinks);
+
+		jsResult = execJavascript(page, "var nodes=[];document.querySelectorAll('h3,a,hr').forEach(function(node){if(node.tagName.toUpperCase()=='A'){nodes.push({'type':'anchor','title':node.innerHTML,'url':node.getAttribute('href'),'add_date':node.getAttribute('add_date'),'last_modified':node.getAttribute('last_modified')})}else if(node.tagName.toUpperCase()=='HR'){nodes.push({'type':'separator'})}else if(node.tagName.toUpperCase()=='H3'){sibling=node.parentNode.querySelector('dl');numChildren=sibling.querySelectorAll(':scope > dt').length;nodes.push({'type':'folder','title':node.innerHTML,'numChildren':numChildren,'add_date':node.getAttribute('add_date'),'last_modified':node.getAttribute('last_modified')})}});nodes");
+		QList<QVariant> items = jsResult.toList();
+
+		processBookmarkItems(items, getImportFolder(), -1);
+		BookmarksManager::getModel()->endImport();
+
+		loop.exit();
+	});
+
+	page.setHtml(QString::fromLatin1(file.readAll()));
+	loop.exec();
+
+	emit importFinished(Importer::BookmarksImport, Importer::SuccessfullImport, m_totalAmount);
+
+	file.close();
+
+	m_isRunning = false;
+
+	deleteLater();
+}
+
+QVariant QtWebEngineBookmarksImportJob::execJavascript(QWebEnginePage &page, const QString &script)
+{
+	QVariant jsResult;
+	QEventLoop loop;
+
+	page.runJavaScript(script, [&loop, &jsResult](const QVariant &result) {
+		jsResult = result;
+		loop.exit(0);
+	});
+	loop.exec();
+
+	return jsResult;
+}
+
+void QtWebEngineBookmarksImportJob::processBookmarkItems(QList<QVariant> &items, BookmarksModel::Bookmark *importFolder, int numChildren)
+{
+	while (numChildren != 0 && m_currentAmount < m_totalAmount)
+	{
+		const QVariant &item = items.at(m_currentAmount++);
+		-- numChildren;
+
+		const QMap<QString, QVariant> itemMap = item.toMap();
+		const QString typeString = itemMap["type"].toString();
+		const QString title = itemMap["title"].toString();
+
+		BookmarksModel::BookmarkType type(BookmarksModel::UnknownBookmark);
+		QMap<int, QVariant> metaData({{BookmarksModel::TitleRole, title}});
+
+		if (typeString == "folder" || typeString == "anchor")
+		{
+			const QDateTime addDate(getDateTime(itemMap["add_date"]));
+			const QDateTime lastModified(getDateTime(itemMap["last_modified"]));
+
+			if (addDate.isValid())
+			{
+				metaData[BookmarksModel::TimeAddedRole] = addDate;
+			}
+
+			if (lastModified.isValid())
+			{
+				metaData[BookmarksModel::TimeAddedRole] = lastModified;
+			}
+		}
+
+		if (typeString == "folder")
+		{
+			processBookmarkItems(items,
+								 BookmarksManager::addBookmark(BookmarksModel::FolderBookmark, metaData, importFolder),
+								 itemMap["numChildren"].toInt());
+		}
+		else
+		{
+			if (typeString == "anchor")
+			{
+				type = BookmarksModel::UrlBookmark;
+				const QString url = itemMap["url"].toString();
+
+				if (!areDuplicatesAllowed() && BookmarksManager::hasBookmark(url))
+				{
+					continue;
+				}
+
+				metaData[BookmarksModel::UrlRole] = url;
+			}
+			else if (typeString == "separator")
+			{
+				type = BookmarksModel::SeparatorBookmark;
+			}
+
+			if (type != BookmarksModel::UnknownBookmark)
+			{
+				BookmarksManager::addBookmark(type, metaData, importFolder);
+			}
+		}
+
+		emit importProgress(Importer::BookmarksImport, m_totalAmount, m_currentAmount);
+	}
+}
+
+void QtWebEngineBookmarksImportJob::cancel()
+{
+}
+
+QDateTime QtWebEngineBookmarksImportJob::getDateTime(const QVariant &attribute)
+{
+#if QT_VERSION < 0x050800
+	const uint seconds(attribute.toUInt());
+
+	return ((seconds > 0) ? QDateTime::fromTime_t(seconds) : QDateTime());
+#else
+	const qint64 seconds(attribute.toLongLong());
+
+	return ((seconds != 0) ? QDateTime::fromSecsSinceEpoch(seconds) : QDateTime());
+#endif
+}
+
+bool QtWebEngineBookmarksImportJob::isRunning() const
+{
+	return m_isRunning;
+}
+
 }

src/modules/backends/web/qtwebengine/QtWebEngineWebBackend.h

@@ -45,6 +45,7 @@ class QtWebEngineWebBackend final : public WebBackend
 	explicit QtWebEngineWebBackend(QObject *parent = nullptr);
 
 	WebWidget* createWidget(const QVariantMap &parameters, ContentsWidget *parent = nullptr) override;
+	BookmarksImportJob* createBookmarksImportJob(BookmarksModel::Bookmark *folder, const QString &path, bool areDuplicatesAllowed) override;
 	QString getName() const override;
 	QString getTitle() const override;
 	QString getDescription() const override;
@@ -80,6 +81,30 @@ protected slots:
 friend class QtWebEnginePage;
 };
 
+class QtWebEngineBookmarksImportJob final : public BookmarksImportJob
+{
+	Q_OBJECT
+
+public:
+	explicit QtWebEngineBookmarksImportJob(BookmarksModel::Bookmark *folder, const QString &path, bool areDuplicatesAllowed, QObject *parent = nullptr);
+	bool isRunning() const override;
+
+public slots:
+	void start() override;
+	void cancel() override;
+
+protected:
+	QVariant execJavascript(QWebEnginePage &page, const QString &script);
+	void processBookmarkItems(QList<QVariant> &items, BookmarksModel::Bookmark *importFolder, int numChildren);
+	static QDateTime getDateTime(const QVariant &attribute);
+
+private:
+	QString m_path;
+	int m_currentAmount;
+	int m_totalAmount;
+	bool m_isRunning;
+};
+
 }
 
 #endif