fix(path): convert to native encoding on Windows

Follow @fxliang 's PR, use `u8path` on Windows to convert UTF-8 string
to Windows native path.
Closes rime#804

Fixes rime/weasel#576
Fixes rime/weasel#1080

BREAKING CHANGE: `installation.yaml` should be UTF-8 encoded.

Previouly on Windows, the file can be written in local encoding to
enable paths with non-ASCII characters. It should be updated to UTF-8
after this change.

Author: lotem

plugins/plugins_module.cc

@@ -20,9 +20,9 @@ namespace rime {
 
 class PluginManager {
  public:
-  void LoadPlugins(fs::path plugins_dir);
+  void LoadPlugins(path plugins_dir);
 
-  static string plugin_name_of(fs::path plugin_file);
+  static string plugin_name_of(path plugin_file);
 
   static PluginManager& instance();
 
@@ -32,14 +32,14 @@ class PluginManager {
   map<string, boost::dll::shared_library> plugin_libs_;
 };
 
-void PluginManager::LoadPlugins(fs::path plugins_dir) {
+void PluginManager::LoadPlugins(path plugins_dir) {
   ModuleManager& mm = ModuleManager::instance();
   if (!fs::is_directory(plugins_dir)) {
     return;
   }
   LOG(INFO) << "loading plugins from " << plugins_dir;
   for (fs::directory_iterator iter(plugins_dir), end; iter != end; ++iter) {
-    fs::path plugin_file = iter->path();
+    path plugin_file = iter->path();
     if (plugin_file.extension() == boost::dll::shared_library::suffix()) {
       fs::file_status plugin_file_status = fs::status(plugin_file);
       if (fs::is_regular_file(plugin_file_status)) {
@@ -69,7 +69,7 @@ void PluginManager::LoadPlugins(fs::path plugins_dir) {
   }
 }
 
-string PluginManager::plugin_name_of(fs::path plugin_file) {
+string PluginManager::plugin_name_of(path plugin_file) {
   string name = plugin_file.stem().string();
   // remove prefix "(lib)rime-"
   if (boost::starts_with(name, "librime-")) {

src/rime/common.h

@@ -9,6 +9,7 @@
 
 #include <rime/build_config.h>
 
+#include <filesystem>
 #include <functional>
 #include <list>
 #include <map>
@@ -80,6 +81,27 @@ inline an<T> New(Args&&... args) {
 using boost::signals2::connection;
 using boost::signals2::signal;
 
+#ifdef _WIN32
+class path : public std::filesystem::path {
+  using fs_path = std::filesystem::path;
+
+ public:
+  path() : fs_path() {}
+  path(const fs_path& p) : fs_path(p) {}
+  path(fs_path&& p) : fs_path(std::move(p)) {}
+  path(const std::string& utf8_path)
+      : fs_path(std::filesystem::u8path(utf8_path))
+  {
+  }
+  path(const char* utf8_path)
+      : fs_path(std::filesystem::u8path(utf8_path))
+  {
+  }
+};
+#else
+using std::filesystem::path;
+#endif
+
 }  // namespace rime
 
 #endif  // RIME_COMMON_H_

src/rime/config/save_output_plugin.cc

@@ -2,7 +2,6 @@
 // Copyright RIME Developers
 // Distributed under the BSD License
 //
-#include <filesystem>
 #include <rime/resource.h>
 #include <rime/service.h>
 #include <rime/config/config_compiler.h>

src/rime/deployer.cc

@@ -7,7 +7,7 @@
 #include <chrono>
 #include <exception>
 #include <utility>
-#include <filesystem>
+#include <rime/common.h>
 #include <rime/deployer.h>
 
 namespace rime {
@@ -147,7 +147,7 @@ void Deployer::JoinMaintenanceThread() {
 }
 
 string Deployer::user_data_sync_dir() const {
-  return (std::filesystem::path(sync_dir) / user_id).string();
+  return (path(sync_dir) / user_id).string();
 }
 
 }  // namespace rime

src/rime/dict/dict_compiler.cc

@@ -22,8 +22,6 @@
 #include <rime/resource.h>
 #include <rime/service.h>
 
-namespace fs = std::filesystem;
-
 namespace rime {
 
 DictCompiler::DictCompiler(Dictionary* dictionary)
@@ -39,7 +37,7 @@ DictCompiler::DictCompiler(Dictionary* dictionary)
 DictCompiler::~DictCompiler() {}
 
 static bool load_dict_settings_from_file(DictSettings* settings,
-                                         const fs::path& dict_file) {
+                                         const path& dict_file) {
   std::ifstream fin(dict_file.string().c_str());
   bool success = settings->LoadDictHeader(fin);
   fin.close();
@@ -53,7 +51,7 @@ static bool get_dict_files_from_settings(vector<string>* dict_files,
     for (auto it = tables->begin(); it != tables->end(); ++it) {
       string dict_name = As<ConfigValue>(*it)->str();
       auto dict_file = source_resolver->ResolvePath(dict_name + ".dict.yaml");
-      if (!fs::exists(dict_file)) {
+      if (!std::filesystem::exists(dict_file)) {
         LOG(ERROR) << "source file '" << dict_file << "' does not exist.";
         return false;
       }
@@ -162,7 +160,7 @@ bool DictCompiler::Compile(const string& schema_file) {
       EntryCollector collector(std::move(syllabary));
       DictSettings settings;
       auto dict_file = source_resolver_->ResolvePath(pack_name + ".dict.yaml");
-      if (!fs::exists(dict_file)) {
+      if (!std::filesystem::exists(dict_file)) {
         LOG(ERROR) << "source file '" << dict_file << "' does not exist.";
         continue;
       }
@@ -188,8 +186,8 @@ bool DictCompiler::Compile(const string& schema_file) {
   return true;
 }
 
-static fs::path relocate_target(const fs::path& source_path,
-                                ResourceResolver* target_resolver) {
+static path relocate_target(const path& source_path,
+                            ResourceResolver* target_resolver) {
   auto resource_id = source_path.filename().string();
   return target_resolver->ResolvePath(resource_id);
 }
@@ -208,7 +206,7 @@ bool DictCompiler::BuildTable(int table_index,
   collector.Configure(settings);
   collector.Collect(dict_files);
   if (options_ & kDump) {
-    fs::path dump_path(table->file_name());
+    path dump_path(table->file_name());
     dump_path.replace_extension(".txt");
     collector.Dump(dump_path.string());
   }
@@ -313,7 +311,7 @@ bool DictCompiler::BuildPrism(const string& schema_file,
     bool enable_correction = false; // Avoid if initializer to comfort compilers
     if (config.GetBool("translator/enable_correction", &enable_correction) &&
         enable_correction) {
-      fs::path corrector_path(prism_->file_name());
+      path corrector_path(prism_->file_name());
       corrector_path.replace_extension("");
       corrector_path.replace_extension(".correction.bin");
       auto target_path = relocate_target(corrector_path,
@@ -331,9 +329,9 @@ bool DictCompiler::BuildPrism(const string& schema_file,
 #endif
   }
   if ((options_ & kDump) && !script.empty()) {
-    fs::path path(prism_->file_name());
-    path.replace_extension(".txt");
-    script.Dump(path.string());
+    path dump_path(prism_->file_name());
+    dump_path.replace_extension(".txt");
+    script.Dump(dump_path.string());
   }
   // build .prism.bin
   {

src/rime/dict/level_db.cc

@@ -5,7 +5,6 @@
 // 2014-12-04 Chen Gong <[email protected]>
 //
 
-#include <filesystem>
 #include <leveldb/db.h>
 #include <leveldb/write_batch.h>
 #include <rime/common.h>

src/rime/dict/preset_vocabulary.cc

@@ -4,7 +4,6 @@
 //
 // 2011-11-27 GONG Chen <[email protected]>
 //
-#include <filesystem>
 #include <utf8.h>
 #include <rime/resource.h>
 #include <rime/service.h>

src/rime/dict/reverse_lookup_dictionary.cc

@@ -9,7 +9,6 @@
 #include <cstdlib>
 #include <sstream>
 #include <boost/algorithm/string.hpp>
-#include <filesystem>
 #include <rime/resource.h>
 #include <rime/schema.h>
 #include <rime/service.h>

src/rime/dict/user_db_recovery_task.cc

@@ -12,8 +12,6 @@
 #include <rime/dict/user_db.h>
 #include <rime/dict/user_db_recovery_task.h>
 
-namespace fs = std::filesystem;
-
 namespace rime {
 
 UserDbRecoveryTask::UserDbRecoveryTask(an<Db> db) : db_(db) {
@@ -65,15 +63,15 @@ void UserDbRecoveryTask::RestoreUserDataFromSnapshot(Deployer* deployer) {
   string dict_name(db_->name());
   boost::erase_last(dict_name, component->extension());
   // locate snapshot file
-  std::filesystem::path dir(deployer->user_data_sync_dir());
+  path dir(deployer->user_data_sync_dir());
   // try *.userdb.txt
-  fs::path snapshot_path = dir / (dict_name + UserDb::snapshot_extension());
-  if (!fs::exists(snapshot_path)) {
+  path snapshot_path = dir / (dict_name + UserDb::snapshot_extension());
+  if (!std::filesystem::exists(snapshot_path)) {
     // try *.userdb.*.snapshot
     string legacy_snapshot_file =
         dict_name + component->extension() + ".snapshot";
     snapshot_path = dir / legacy_snapshot_file;
-    if (!fs::exists(snapshot_path)) {
+    if (!std::filesystem::exists(snapshot_path)) {
       return;  // not found
     }
   }

src/rime/gear/simplifier.cc

@@ -5,7 +5,6 @@
 // 2011-12-12 GONG Chen <[email protected]>
 //
 #include <boost/algorithm/string.hpp>
-#include <filesystem>
 #include <stdint.h>
 #include <utf8.h>
 #include <utility>
@@ -25,11 +24,6 @@
 #include <opencc/Dict.hpp>
 #include <opencc/DictEntry.hpp>
 
-#ifdef _MSC_VER
-#include <opencc/UTF8Util.hpp>
-namespace fs = std::filesystem;
-#endif
-
 static const char* quote_left = "\xe3\x80\x94";   //"\xef\xbc\x88";
 static const char* quote_right = "\xe3\x80\x95";  //"\xef\xbc\x89";
 
@@ -42,13 +36,9 @@ class Opencc {
     opencc::Config config;
     try {
       // windows config_path in CP_ACP, convert it to UTF-8
-#ifdef _MSC_VER
-      fs::path path{config_path};
-      converter_ =
-          config.NewFromFile(opencc::UTF8Util::U16ToU8(path.wstring()));
-#else
+      path path{config_path};
       converter_ = config.NewFromFile(config_path);
-#endif /*  _MSC_VER */
+
       const list<opencc::ConversionPtr> conversions =
           converter_->GetConversionChain()->GetConversions();
       dict_ = conversions.front()->GetDict();
@@ -178,7 +168,6 @@ Simplifier::Simplifier(const Ticket& ticket)
 }
 
 void Simplifier::Initialize() {
-  using namespace std::filesystem;
   initialized_ = true;  // no retry
   path opencc_config_path = opencc_config_;
   if (opencc_config_path.extension().string() == ".ini") {

src/rime/lever/custom_settings.cc

@@ -5,14 +5,11 @@
 // 2012-02-26 GONG Chen <[email protected]>
 //
 #include <boost/algorithm/string.hpp>
-#include <filesystem>
 #include <rime/config.h>
 #include <rime/deployer.h>
 #include <rime/signature.h>
 #include <rime/lever/custom_settings.h>
 
-namespace fs = std::filesystem;
-
 namespace rime {
 
 static string remove_suffix(const string& input, const string& suffix) {
@@ -31,17 +28,15 @@ CustomSettings::CustomSettings(Deployer* deployer,
     : deployer_(deployer), config_id_(config_id), generator_id_(generator_id) {}
 
 bool CustomSettings::Load() {
-  fs::path config_path =
-      fs::path(deployer_->staging_dir) / (config_id_ + ".yaml");
+  path config_path = path(deployer_->staging_dir) / (config_id_ + ".yaml");
   if (!config_.LoadFromFile(config_path.string())) {
-    config_path =
-        fs::path(deployer_->prebuilt_data_dir) / (config_id_ + ".yaml");
+    config_path = path(deployer_->prebuilt_data_dir) / (config_id_ + ".yaml");
     if (!config_.LoadFromFile(config_path.string())) {
       LOG(WARNING) << "cannot find '" << config_id_ << ".yaml'.";
     }
   }
-  fs::path custom_config_path =
-      fs::path(deployer_->user_data_dir) / custom_config_file(config_id_);
+  path custom_config_path =
+      path(deployer_->user_data_dir) / custom_config_file(config_id_);
   if (!custom_config_.LoadFromFile(custom_config_path.string())) {
     return false;
   }
@@ -54,7 +49,7 @@ bool CustomSettings::Save() {
     return false;
   Signature signature(generator_id_, "customization");
   signature.Sign(&custom_config_, deployer_);
-  fs::path custom_config_path(deployer_->user_data_dir);
+  path custom_config_path(deployer_->user_data_dir);
   custom_config_path /= custom_config_file(config_id_);
   custom_config_.SaveToFile(custom_config_path.string());
   modified_ = false;
@@ -87,7 +82,7 @@ bool CustomSettings::Customize(const string& key, const an<ConfigItem>& item) {
 }
 
 bool CustomSettings::IsFirstRun() {
-  fs::path custom_config_path(deployer_->user_data_dir);
+  path custom_config_path(deployer_->user_data_dir);
   custom_config_path /= custom_config_file(config_id_);
   Config config;
   if (!config.LoadFromFile(custom_config_path.string()))

src/rime/lever/customizer.cc

@@ -4,6 +4,7 @@
 //
 // 2011-12-12 GONG Chen <[email protected]>
 //
+#include <filesystem>
 #include <stdint.h>
 #include <rime/common.h>
 #include <rime/config.h>
@@ -60,7 +61,7 @@ bool Customizer::UpdateConfigFile() {
     }
   }
 
-  fs::path custom_path(dest_path_);
+  path custom_path(dest_path_);
   if (custom_path.extension() != ".yaml") {
     custom_path.clear();
   } else {

src/rime/lever/customizer.h

@@ -5,14 +5,14 @@
 #ifndef RIME_CUSTOMIZER_H_
 #define RIME_CUSTOMIZER_H_
 
-#include <filesystem>
+#include <rime/common.h>
 
 namespace rime {
 
 class Customizer {
  public:
-  Customizer(const std::filesystem::path& source_path,
-             const std::filesystem::path& dest_path,
+  Customizer(const path& source_path,
+             const path& dest_path,
              const string& version_key)
       : source_path_(source_path),
         dest_path_(dest_path),
@@ -22,8 +22,8 @@ class Customizer {
   bool UpdateConfigFile();
 
  protected:
-  std::filesystem::path source_path_;
-  std::filesystem::path dest_path_;
+  path source_path_;
+  path dest_path_;
   string version_key_;
 };