Saurabh95/encoding support (#13412)

* Now encoding is passed as parameter on file read and it is used by writefile in order to preserve encoding

* UI wiring

* Code Cleanup

* Fixed failing tests

* Added warning Dialog while changing encoding

* Added some Linux specific changes

* Fixed some linting errors

* Reverted last commit

* Minor changes

* Now selected encoding is stored in state

* Fixed lint error

* Added some more encodings

* Added some more encodings

* Fixed lint errors

* Removed duplicate encodings

* Fixed failing tests

* Used externalized strings

* Addressed review comments

* Added supported encodings file

* Addressed review comments

* Addressed review comments

Author: saurabh95

src/document/Document.js

@@ -758,6 +758,25 @@ define(function (require, exports, module) {
         return this.file instanceof InMemoryFile;
     };
 
+    /**
+     *  Reloads the document from FileSystem
+     *  @return {promise} - to check if reload was successful or not
+     */
+    Document.prototype.reload = function () {
+        var $deferred = $.Deferred();
+        var self = this;
+        FileUtils.readAsText(this.file)
+            .done(function (text, readTimestamp) {
+                self.refreshText(text, readTimestamp);
+                $deferred.resolve();
+            })
+            .fail(function (error) {
+                console.log("Error reloading contents of " + self.file.fullPath, error);
+                $deferred.reject();
+            });
+        return $deferred.promise();
+    };
+
     // We dispatch events from the module level, and the instance level. Instance events are wired up
     // in the Document constructor.
     EventDispatcher.makeEventDispatcher(exports);

src/document/DocumentCommandHandlers.js

@@ -322,6 +322,22 @@ define(function (require, exports, module) {
             });
 
             var file = FileSystem.getFileForPath(fullPath);
+            if (options && options.encoding) {
+                file._encoding = options.encoding;
+            } else {
+                var projectRoot = ProjectManager.getProjectRoot(),
+                    context = {
+                        location : {
+                            scope: "user",
+                            layer: "project",
+                            layerID: projectRoot.fullPath
+                        }
+                    };
+                var encoding = PreferencesManager.getViewState("encoding", context);
+                if (encoding[fullPath]) {
+                    file._encoding = encoding[fullPath];
+                }
+            }
             MainViewManager._open(paneId, file, options)
                 .done(function () {
                     result.resolve(file);
@@ -902,7 +918,21 @@ define(function (require, exports, module) {
             doc.isSaving = true;    // mark that we're saving the document
 
             // First, write document's current text to new file
+            if (doc.file._encoding && doc.file._encoding !== "UTF-8") {
+                var projectRoot = ProjectManager.getProjectRoot(),
+                    context = {
+                        location : {
+                            scope: "user",
+                            layer: "project",
+                            layerID: projectRoot.fullPath
+                        }
+                    };
+                var encoding = PreferencesManager.getViewState("encoding", context);
+                encoding[path] = doc.file._encoding;
+                PreferencesManager.setViewState("encoding", encoding, context);
+            }
             newFile = FileSystem.getFileForPath(path);
+            newFile._encoding = doc.file._encoding;
 
             // Save as warns you when you're about to overwrite a file, so we
             // explicitly allow "blind" writes to the filesystem in this case,

src/document/DocumentManager.js

@@ -93,6 +93,7 @@ define(function (require, exports, module) {
         Commands            = require("command/Commands"),
         PerfUtils           = require("utils/PerfUtils"),
         LanguageManager     = require("language/LanguageManager"),
+        ProjectManager      = require("project/ProjectManager"),
         Strings             = require("strings");
 
 
@@ -419,7 +420,7 @@ define(function (require, exports, module) {
         if (doc) {
             result.resolve(doc.getText(), doc.diskTimestamp, checkLineEndings ? doc._lineEndings : null);
         } else {
-            file.read(function (err, contents, stat) {
+            file.read(function (err, contents, encoding, stat) {
                 if (err) {
                     result.reject(err);
                 } else {
@@ -498,6 +499,18 @@ define(function (require, exports, module) {
         //  via notifyFileDeleted
         FileSyncManager.syncOpenDocuments(Strings.FILE_DELETED_TITLE);
 
+        var projectRoot = ProjectManager.getProjectRoot(),
+            context = {
+                location : {
+                    scope: "user",
+                    layer: "project",
+                    layerID: projectRoot.fullPath
+                }
+            };
+        var encoding = PreferencesManager.getViewState("encoding", context);
+        delete encoding[fullPath];
+        PreferencesManager.setViewState("encoding", encoding, context);
+
         if (!getOpenDocumentForPath(fullPath) &&
                 !MainViewManager.findInAllWorkingSets(fullPath).length) {
             // For images not open in the workingset,

src/editor/EditorStatusBar.js

@@ -40,10 +40,21 @@ define(function (require, exports, module) {
         PreferencesManager   = require("preferences/PreferencesManager"),
         StatusBar            = require("widgets/StatusBar"),
         Strings              = require("strings"),
+        FileUtils            = require("file/FileUtils"),
+        InMemoryFile         = require("document/InMemoryFile"),
+        Dialogs              = require("widgets/Dialogs"),
+        DefaultDialogs       = require("widgets/DefaultDialogs"),
+        ProjectManager       = require("project/ProjectManager"),
+        Async                = require("utils/Async"),
+        FileSystem           = require("filesystem/FileSystem"),
         StringUtils          = require("utils/StringUtils");
+    
+    var SupportedEncodingsText = require("text!supported-encodings.json"),
+        SupportedEncodings = JSON.parse(SupportedEncodingsText);
 
     /* StatusBar indicators */
     var languageSelect, // this is a DropdownButton instance
+        encodingSelect, // this is a DropdownButton instance
         $cursorInfo,
         $fileInfo,
         $indentType,
@@ -74,12 +85,24 @@ define(function (require, exports, module) {
         var doc = editor.document,
             lang = doc.getLanguage();
 
-        // Ensure width isn't left locked by a previous click of the dropdown (which may not have resulted in a "change" event at the time)
-        languageSelect.$button.css("width", "auto");
         // Show the current language as button title
         languageSelect.$button.text(lang.getName());
     }
 
+    /**
+     * Update encoding
+     * @param {Editor} editor Current editor
+     */
+    function _updateEncodingInfo(editor) {
+        var doc = editor.document;
+
+        // Show the current encoding as button title
+        if (!doc.file._encoding) {
+            doc.file._encoding = "UTF-8";
+        }
+        encodingSelect.$button.text(doc.file._encoding);
+    }
+
     /**
      * Update file information
      * @param {Editor} editor Current editor
@@ -277,6 +300,7 @@ define(function (require, exports, module) {
 
             _updateCursorInfo(null, current);
             _updateLanguageInfo(current);
+            _updateEncodingInfo(current);
             _updateFileInfo(current);
             _initOverwriteMode(current);
             _updateIndentType(fullPath);
@@ -305,6 +329,40 @@ define(function (require, exports, module) {
         languageSelect.items.unshift(LANGUAGE_SET_AS_DEFAULT);
     }
 
+    /**
+     * Change the encoding and reload the current document.
+     * If passed then save the preferred encoding in state.
+     */
+    function _changeEncodingAndReloadDoc(document) {
+        var promise = document.reload();
+        promise.done(function (text, readTimestamp) {
+            encodingSelect.$button.text(document.file._encoding);
+            // Store the preferred encoding in the state
+            var projectRoot = ProjectManager.getProjectRoot(),
+                context = {
+                    location : {
+                        scope: "user",
+                        layer: "project",
+                        layerID: projectRoot.fullPath
+                    }
+                };
+            var encoding = PreferencesManager.getViewState("encoding", context);
+            encoding[document.file.fullPath] = document.file._encoding;
+            PreferencesManager.setViewState("encoding", encoding, context);
+        });
+        promise.fail(function (error) {
+            console.log("Error reloading contents of " + document.file.fullPath, error);
+        });
+    }
+
+
+    /**
+     * Populate the encodingSelect DropdownButton's menu with all registered encodings
+     */
+    function _populateEncodingDropdown() {
+        encodingSelect.items = SupportedEncodings;
+    }
+
     /**
      * Initialize
      */
@@ -343,6 +401,27 @@ define(function (require, exports, module) {
         $("#status-language").append(languageSelect.$button);
         languageSelect.$button.attr("title", Strings.STATUSBAR_LANG_TOOLTIP);
 
+
+        encodingSelect = new DropdownButton("", [], function (item, index) {
+            var document = EditorManager.getActiveEditor().document;
+            var html = _.escape(item);
+
+            // Show indicators for currently selected & default languages for the current file
+            if (item === "UTF-8") {
+                html += " <span class='default-language'>" + Strings.STATUSBAR_DEFAULT_LANG + "</span>";
+            }
+            if (item === document.file._encoding) {
+                html = "<span class='checked-language'></span>" + html;
+            }
+            return html;
+        });
+
+        encodingSelect.dropdownExtraClasses = "dropdown-status-bar";
+        encodingSelect.$button.addClass("btn-status-bar");
+        $("#status-encoding").append(encodingSelect.$button);
+        encodingSelect.$button.attr("title", Strings.STATUSBAR_ENCODING_TOOLTIP);
+
+
         // indentation event handlers
         $indentType.on("click", _toggleIndentType);
         $indentWidthLabel
@@ -389,16 +468,96 @@ define(function (require, exports, module) {
             }
         });
 
+        // Encoding select change handler
+        encodingSelect.on("select", function (e, encoding) {
+            var document = EditorManager.getActiveEditor().document,
+                fullPath = document.file.fullPath;
+
+            document.file._encoding = encoding;
+            
+
+            if (!(document.file instanceof InMemoryFile) && document.isDirty) {
+                var dialogId = DefaultDialogs.DIALOG_ID_EXT_CHANGED,
+                    message = StringUtils.format(
+                        Strings.DIRTY_FILE_ENCODING_CHANGE_WARN,
+                        StringUtils.breakableUrl(
+                            ProjectManager.makeProjectRelativeIfPossible(document.file.fullPath)
+                        )
+                    ),
+                    buttons = [
+                        {
+                            className: Dialogs.DIALOG_BTN_CLASS_LEFT,
+                            id:        Dialogs.DIALOG_BTN_DONTSAVE,
+                            text:      Strings.IGNORE_RELOAD_FROM_DISK
+                        },
+                        {
+                            className: Dialogs.DIALOG_BTN_CLASS_PRIMARY,
+                            id:        Dialogs.DIALOG_BTN_CANCEL,
+                            text:      Strings.CANCEL
+                        }
+                    ];
+                
+                Dialogs.showModalDialog(dialogId, Strings.SAVE_FILE_ENCODING_CHANGE_WARN, message, buttons)
+                    .done(function (id) {
+                        if (id === Dialogs.DIALOG_BTN_DONTSAVE) {
+                            _changeEncodingAndReloadDoc(document);
+                        }
+                    });
+            } else if (document.file instanceof InMemoryFile) {
+                encodingSelect.$button.text(encoding);
+            } else if (!document.isDirty) {
+                _changeEncodingAndReloadDoc(document);
+            }
+        });
+
         $statusOverwrite.on("click", _updateEditorOverwriteMode);
     }
 
     // Initialize: status bar focused listener
     EditorManager.on("activeEditorChange", _onActiveEditorChange);
 
+    function _checkFileExistance(filePath, index, encoding) {
+        var deferred = new $.Deferred(),
+            fileEntry = FileSystem.getFileForPath(filePath);
+    
+        fileEntry.exists(function (err, exists) {
+            if (!err && exists) {
+                deferred.resolve();
+            } else {
+                delete encoding[filePath];
+                deferred.reject();
+            }
+        });
+
+        return deferred.promise();
+    }
+
+    ProjectManager.on("projectOpen", function () {
+        var projectRoot = ProjectManager.getProjectRoot(),
+            context = {
+                location : {
+                    scope: "user",
+                    layer: "project",
+                    layerID: projectRoot.fullPath
+                }
+            };
+        var encoding = PreferencesManager.getViewState("encoding", context);
+        if (!encoding) {
+            PreferencesManager.setViewState("encoding", {}, context);
+        }
+        Async.doSequentially(Object.keys(encoding), function (filePath, index) {
+            return _checkFileExistance(filePath, index, encoding);
+        }, false)
+            .always(function () {
+                PreferencesManager.setViewState("encoding", encoding, context);
+            });
+    });
+
     AppInit.htmlReady(_init);
     AppInit.appReady(function () {
         // Populate language switcher with all languages after startup; update it later if this set changes
         _populateLanguageDropdown();
+        _populateEncodingDropdown();
         LanguageManager.on("languageAdded languageModified", _populateLanguageDropdown);
         _onActiveEditorChange(null, EditorManager.getActiveEditor(), null);
         StatusBar.show();

src/file/FileUtils.js

@@ -77,7 +77,7 @@ define(function (require, exports, module) {
         });
 
         // Read file
-        file.read(function (err, data, stat) {
+        file.read(function (err, data, encoding, stat) {
             if (!err) {
                 result.resolve(data, stat.mtime);
             } else {