Support unicode characters in markdown comments

- convert unicode chars in CoreMarkdownAccessImpl.removeDocLineIntros()
- add new test to MarkdownCommentTests
- fixes eclipse-jdt#1787

Author: jjohnstn

org.eclipse.jdt.core.manipulation/common/org/eclipse/jdt/core/manipulation/internal/javadoc/CoreMarkdownAccessImpl.java

@@ -16,6 +16,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.regex.Pattern;
 
 import org.commonmark.Extension;
 import org.commonmark.ext.gfm.tables.TablesExtension;
@@ -52,12 +53,17 @@ private void init() {
 		fRenderer= HtmlRenderer.builder().extensions(extensions).build();
 	}
 
+	final static Pattern UnicodePattern= Pattern.compile("\\\\u([0-9A-Fa-f]{4})"); //$NON-NLS-1$
+
 	@Override
 	protected String removeDocLineIntros(String textWithSlashes) {
+		// replace unicode characters
+		String content= UnicodePattern.matcher(textWithSlashes).replaceAll(s -> String.valueOf((char)Integer.parseInt(s.group(1), 16)));
+
 		String lineBreakGroup= "(\\r\\n?|\\n)"; //$NON-NLS-1$
 		String noBreakSpace= "[^\r\n&&\\s]"; //$NON-NLS-1$
 		// in the markdown case relevant leading whitespace is contained in TextElements, no need to preserve blanks *between* elements
-		return textWithSlashes.replaceAll(lineBreakGroup + noBreakSpace + "*///" + noBreakSpace + '*', "$1"); //$NON-NLS-1$ //$NON-NLS-2$
+		return content.replaceAll(lineBreakGroup + noBreakSpace + "*///" + noBreakSpace + '*', "$1"); //$NON-NLS-1$ //$NON-NLS-2$
 	}
 
 	@Override

org.eclipse.jdt.ui.tests/ui/org/eclipse/jdt/ui/tests/hover/MarkdownCommentTests.java

@@ -900,6 +900,30 @@ <p>This is a test Javadoc for method test4()
 		assertHtmlContent(expectedContent, actualHtmlContent);
 	}
 	@Test
+	public void testGH1787() throws CoreException {
+		String source= """
+				package p;
+
+				public class E {
+				    /// Unicode in markdown \u000A///\u000D///\u000D\u000A///here
+				    public void m() {}
+				}
+
+				""";
+		ICompilationUnit cu= getWorkingCopy("/TestSetupProject/src/p/E.java", source, null);
+		assertNotNull("E.java", cu);
+
+		IType type= cu.getType("E");
+
+		IMethod method= type.getMethods()[0];
+		String actualHtmlContent= getHoverHtmlContent(cu, method);
+		assertHtmlContent("""
+				<p>Unicode in markdown</p>
+				<p>here</p>
+				""",
+				actualHtmlContent);
+	}
+	@Test
 	public void testFenceLenFour_1() throws CoreException {
 		String source= """
 				/// ````