Skip to content

Commit f116900

Browse files
KvanTTTKvanTTT
authored
Merge pull request #116 from PositiveTechnologies/dev
Location conversion & Json improvements, bugfixes & refactoring
2 parents 4263e45 + e0e942a commit f116900

File tree

109 files changed

+930
-807
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

109 files changed

+930
-807
lines changed

‎PT.PM.Cli.nuspec

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
<package xmlns="http://schemas.microsoft.com/packaging/2010/07/nuspec.xsd">
33
<metadata>
44
<id>PT.PM.Cli</id>
5-
<version>1.5</version>
5+
<version>1.5.1</version>
66
<description>An engine with CLI for searching patterns in the source code, based on Unified AST or UST. At present time C#, Java, PHP, PL/SQL, T-SQL, and JavaScript are supported. Patterns can be described within the code or using a DSL.</description>
77
<authors>Positive Technologies</authors>
88

‎PT.PM.nuspec

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
<package xmlns="http://schemas.microsoft.com/packaging/2010/07/nuspec.xsd">
33
<metadata>
44
<id>PT.PM</id>
5-
<version>1.5</version>
5+
<version>1.5.1</version>
66
<description>An engine for searching patterns in the source code, based on Unified AST or UST. At present time C#, Java, PHP, PL/SQL, T-SQL, and JavaScript are supported. Patterns can be described within the code or using a DSL.</description>
77
<authors>Positive Technologies</authors>
88

‎Sources/AspxParser

‎Sources/PT.PM.AntlrUtils/AntlrConverter.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ public RootUst Convert(ParseTree langParseTree)
6666
}
6767
catch (Exception ex)
6868
{
69-
Logger.LogError(new ConversionException(filePath, ex));
69+
Logger.LogError(new ConversionException(langParseTree.SourceCodeFile, ex));
7070

7171
if (result == null)
7272
{
@@ -103,7 +103,7 @@ public Ust Visit(IParseTree tree)
103103
{
104104
if (tree is ParserRuleContext parserRuleContext)
105105
{
106-
Logger.LogConversionError(ex, parserRuleContext, root.SourceCodeFile.RelativeName, root.SourceCodeFile.Code);
106+
Logger.LogConversionError(ex, parserRuleContext, root.SourceCodeFile);
107107
}
108108
return DefaultResult;
109109
}

‎Sources/PT.PM.AntlrUtils/AntlrError.cs

Lines changed: 0 additions & 28 deletions
This file was deleted.

‎Sources/PT.PM.AntlrUtils/AntlrLexerError.cs

Lines changed: 0 additions & 20 deletions
This file was deleted.

‎Sources/PT.PM.AntlrUtils/AntlrMemoryErrorListener.cs

Lines changed: 19 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,9 @@ namespace PT.PM.AntlrUtils
66
{
77
public class AntlrMemoryErrorListener : IAntlrErrorListener<IToken>, IAntlrErrorListener<int>
88
{
9-
private const int MaxErrorCodeLength = 200;
10-
private const string ErrorCodeSplitter = " ... ";
11-
129
public ILogger Logger { get; set; } = DummyLogger.Instance;
1310

14-
public string FileName { get; set; }
15-
16-
public string FileData { get; set; }
11+
public CodeFile CodeFile { get; set; }
1712

1813
public bool IsPattern { get; set; }
1914

@@ -25,32 +20,34 @@ public AntlrMemoryErrorListener()
2520

2621
public void SyntaxError(IRecognizer recognizer, int offendingSymbol, int line, int charPositionInLine, string msg, RecognitionException e)
2722
{
28-
var error = new AntlrLexerError(offendingSymbol, line, charPositionInLine, msg, e);
29-
int start = TextUtils.LineColumnToLinear(FileData, line, charPositionInLine);
30-
string errorText = FixLineNumber(error.ToString(), line, charPositionInLine);
31-
Logger.LogError(new ParsingException(FileName, message: errorText) { TextSpan = new TextSpan(start, 1), IsPattern = IsPattern });
23+
if (recognizer is Lexer lexer)
24+
{
25+
ProcessError(lexer.CharIndex, lexer.CharIndex, msg);
26+
}
3227
}
3328

3429
public void SyntaxError(IRecognizer recognizer, IToken offendingSymbol, int line, int charPositionInLine, string msg, RecognitionException e)
3530
{
36-
var error = new AntlrParserError(offendingSymbol, line, charPositionInLine, msg, e);
37-
string errorText = FixLineNumber(error.ToString(), line, charPositionInLine);
38-
int start = TextUtils.LineColumnToLinear(FileData, line, charPositionInLine);
39-
Logger.LogError(new ParsingException(FileName, message: errorText) { TextSpan = new TextSpan(start, 1), IsPattern = IsPattern });
31+
ProcessError(offendingSymbol.StartIndex, offendingSymbol.StopIndex, msg);
4032
}
4133

42-
private string FixLineNumber(string errorText, int line, int charPositionInLine)
34+
private void ProcessError(int startIndex, int stopIndex, string msg)
4335
{
44-
if (LineOffset != 0)
36+
int lineLinearIndex = CodeFile.GetLineLinearIndex(LineOffset);
37+
startIndex = startIndex + lineLinearIndex;
38+
stopIndex = stopIndex + 1 + lineLinearIndex;
39+
if (stopIndex <= startIndex)
4540
{
46-
int atLastIndexOf = errorText.LastIndexOf("at");
47-
if (atLastIndexOf != -1)
48-
{
49-
errorText = errorText.Remove(atLastIndexOf) + $"at {LineOffset + line}:{charPositionInLine}";
50-
}
41+
startIndex = stopIndex - 1;
5142
}
43+
TextSpan textSpan = TextSpan.FromBounds(startIndex, stopIndex);
44+
45+
string errorMessage = $"{msg} at {CodeFile.GetLineColumnTextSpan(textSpan)}";
5246

53-
return errorText;
47+
Logger.LogError(new ParsingException(CodeFile, message: errorMessage)
48+
{
49+
TextSpan = textSpan,
50+
});
5451
}
5552
}
5653
}

‎Sources/PT.PM.AntlrUtils/AntlrParser.cs

Lines changed: 12 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,19 @@
1-
using PT.PM.Common;
2-
using Antlr4.Runtime;
1+
using Antlr4.Runtime;
32
using Antlr4.Runtime.Atn;
43
using Antlr4.Runtime.Misc;
4+
using PT.PM.Common;
5+
using PT.PM.Common.Exceptions;
56
using System;
67
using System.Collections.Generic;
78
using System.Diagnostics;
8-
using System.IO;
99
using System.Text;
1010
using System.Threading;
11-
using PT.PM.Common.Exceptions;
1211

1312
namespace PT.PM.AntlrUtils
1413
{
1514
public abstract class AntlrParser : ILanguageParser
1615
{
17-
private int processedFilesCount = 1;
16+
private static int processedFilesCount = 1;
1817
private static ReaderWriterLockSlim lexerLock = new ReaderWriterLockSlim();
1918
private static ReaderWriterLockSlim parserLock = new ReaderWriterLockSlim();
2019

@@ -56,16 +55,15 @@ public AntlrParser()
5655
Parser = InitParser(null);
5756
}
5857

59-
public ParseTree Parse(SourceCodeFile sourceCodeFile)
58+
public ParseTree Parse(CodeFile sourceCodeFile)
6059
{
6160
AntlrParseTree result = null;
6261

63-
var filePath = Path.Combine(sourceCodeFile.RelativePath, sourceCodeFile.Name);
62+
var filePath = sourceCodeFile.RelativeName;
6463
if (sourceCodeFile.Code != null)
6564
{
6665
var errorListener = new AntlrMemoryErrorListener();
67-
errorListener.FileName = filePath;
68-
errorListener.FileData = sourceCodeFile.Code;
66+
errorListener.CodeFile = sourceCodeFile;
6967
errorListener.Logger = Logger;
7068
errorListener.LineOffset = LineOffset;
7169
try
@@ -120,7 +118,7 @@ public ParseTree Parse(SourceCodeFile sourceCodeFile)
120118
}
121119
catch (Exception ex)
122120
{
123-
Logger.LogError(new ParsingException(filePath, ex));
121+
Logger.LogError(new ParsingException(sourceCodeFile, ex));
124122

125123
if (result == null)
126124
{
@@ -139,11 +137,11 @@ public ParseTree Parse(SourceCodeFile sourceCodeFile)
139137

140138
public void ClearCache()
141139
{
142-
ClearCacheIfRequired(InitLexer(null).Interpreter, lexerLock, 1);
140+
ClearCacheIfRequired(InitLexer(null).Interpreter, lexerLock, 1, false);
143141
ClearCacheIfRequired(InitParser(null).Interpreter, parserLock, 1);
144142
}
145143

146-
protected ParserRuleContext ParseTokens(SourceCodeFile sourceCodeFile,
144+
protected ParserRuleContext ParseTokens(CodeFile sourceCodeFile,
147145
AntlrMemoryErrorListener errorListener, BufferedTokenStream codeTokenStream,
148146
Func<ITokenStream, Parser> initParserFunc = null, Func<Parser, ParserRuleContext> parseFunc = null)
149147
{
@@ -203,7 +201,7 @@ protected ParserRuleContext ParseTokens(SourceCodeFile sourceCodeFile,
203201
/// </summary>
204202
/// <param name="file"></param>
205203
/// <returns></returns>
206-
protected virtual string PreprocessText(SourceCodeFile file)
204+
protected virtual string PreprocessText(CodeFile file)
207205
{
208206
var text = file.Code;
209207
var result = new StringBuilder(text.Length);
@@ -277,10 +275,7 @@ protected void ClearCacheIfRequired(ATNSimulator interpreter, ReaderWriterLockSl
277275
protected void IncrementProcessedFilesCount()
278276
{
279277
int newValue = Interlocked.Increment(ref processedFilesCount);
280-
if (newValue == int.MaxValue)
281-
{
282-
processedFilesCount = 1;
283-
}
278+
processedFilesCount = newValue == int.MaxValue ? 1 : newValue;
284279
}
285280
}
286281
}

‎Sources/PT.PM.AntlrUtils/AntlrParserError.cs

Lines changed: 0 additions & 15 deletions
This file was deleted.

‎Sources/PT.PM.AntlrUtils/AntlrUtils.cs

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ public static TextSpan GetTextSpan(this IToken token)
6767
}
6868

6969
public static void LogConversionError(this ILogger logger, Exception ex,
70-
ParserRuleContext context, string currentFileName, string currentFileData)
70+
ParserRuleContext context, CodeFile currentFileData)
7171
{
7272
StackTrace stackTrace = new StackTrace(ex, true);
7373
int frameNumber = 0;
@@ -88,17 +88,16 @@ public static void LogConversionError(this ILogger logger, Exception ex,
8888

8989
var textSpan = context.GetTextSpan();
9090
string exceptionText;
91-
int sourceCodeLine, sourceCodeColumn;
92-
textSpan.Start.ToLineColumn(currentFileData, out sourceCodeLine, out sourceCodeColumn);
91+
LineColumnTextSpan lineColumnTextSpan = currentFileData.GetLineColumnTextSpan(textSpan);
9392
if (fileName != null)
9493
{
95-
exceptionText = $"{ex.Message} at method \"{methodName}\" {line}:{column} at position {sourceCodeLine}:{sourceCodeColumn} in source file";
94+
exceptionText = $"{ex.Message} at method \"{methodName}\" {line}:{column} at position {lineColumnTextSpan.BeginLine}:{lineColumnTextSpan.BeginColumn} in source file";
9695
}
9796
else
9897
{
99-
exceptionText = $"{ex.Message} at position {sourceCodeLine}:{sourceCodeColumn} in source file";
98+
exceptionText = $"{ex.Message} at position {lineColumnTextSpan.BeginLine}:{lineColumnTextSpan.BeginColumn} in source file";
10099
}
101-
logger.LogError(new ConversionException(currentFileName, message: exceptionText) { TextSpan = textSpan });
100+
logger.LogError(new ConversionException(currentFileData, message: exceptionText) { TextSpan = textSpan });
102101
}
103102
}
104103
}

‎Sources/PT.PM.AntlrUtils/PT.PM.AntlrUtils.csproj

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -48,9 +48,6 @@
4848
<Compile Include="AntlrUtils.cs" />
4949
<Compile Include="AntlrMemoryErrorListener.cs" />
5050
<Compile Include="AntlrParser.cs" />
51-
<Compile Include="AntlrError.cs" />
52-
<Compile Include="AntlrParserError.cs" />
53-
<Compile Include="AntlrLexerError.cs" />
5451
<Compile Include="AntlrConverter.cs" />
5552
<Compile Include="CaseInsensitiveType.cs" />
5653
<Compile Include="Properties\AssemblyInfo.cs" />

‎Sources/PT.PM.CSharpParseTreeUst.Tests/AspxTests.cs

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,15 +20,17 @@ public void Convert_AspxLineColumnPosition_Correct()
2020
AspxParseResult result = aspxParser.Parse(source);
2121
var foundNode = result.RootNode.Descendants<AspxNode.AspxExpressionTag>()
2222
.FirstOrDefault(node => node.Expression.Contains("Expression text"));
23-
foundNode.Location.Start.ToLineColumn(source.Text, out int line, out int column);
23+
24+
var sourceCode = new CodeFile(source.Text);
25+
sourceCode.GetLineColumnFromLinear(foundNode.Location.Start, out int line, out int column);
2426
Assert.AreEqual(15, line);
2527
Assert.AreEqual(13, column);
26-
Assert.AreEqual(foundNode.Location.Start, TextUtils.LineColumnToLinear(source.Text, line, column));
28+
Assert.AreEqual(foundNode.Location.Start, sourceCode.GetLinearFromLineColumn(line, column));
2729

28-
foundNode.Location.End.ToLineColumn(source.Text, out line, out column);
30+
sourceCode.GetLineColumnFromLinear(foundNode.Location.End, out line, out column);
2931
Assert.AreEqual(15, line);
3032
Assert.AreEqual(30, column);
31-
Assert.AreEqual(foundNode.Location.End, TextUtils.LineColumnToLinear(source.Text, line, column));
33+
Assert.AreEqual(foundNode.Location.End, sourceCode.GetLinearFromLineColumn(line, column));
3234
}
3335

3436
[TestCase("TestAspxParser.aspx")]

‎Sources/PT.PM.CSharpParseTreeUst/Aspx.cs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ namespace PT.PM.CSharpParseTreeUst
55
public static class Aspx
66
{
77
public readonly static Language Language =
8-
new Language("Aspx", new[] { ".asax", ".aspx", ".ascx", ".master" }, false, "Aspx", new[] { CSharp.Language }, isPattern: false);
8+
new Language("Aspx", new[] { ".asax", ".aspx", ".ascx", ".master" },
9+
false, "Aspx", new[] { CSharp.Language }, false, false);
910
}
1011
}

‎Sources/PT.PM.CSharpParseTreeUst/AspxConverter.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ namespace PT.PM.CSharpParseTreeUst
1515
public class AspxConverter : DepthFirstAspxWithoutCloseTagVisitor<Ust>, IParseTreeToUstConverter
1616
{
1717
private Stack<bool> runAtServer = new Stack<bool>();
18-
private SourceCodeFile sourceCodeFile;
18+
private CodeFile sourceCodeFile;
1919
private int namespaceDepth;
2020

2121
public Language Language => Aspx.Language;
@@ -56,7 +56,7 @@ public RootUst Convert(ParseTree langParseTree)
5656
}
5757
catch (Exception ex)
5858
{
59-
Logger.LogError(new ConversionException(aspxParseTree.SourceCodeFile.FullName, ex));
59+
Logger.LogError(new ConversionException(aspxParseTree.SourceCodeFile, ex));
6060
result = new RootUst(langParseTree.SourceCodeFile, Language);
6161
}
6262

‎Sources/PT.PM.CSharpParseTreeUst/AspxParser.cs

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ public class AspxParser : ILanguageParser
1212

1313
public Language Language => Aspx.Language;
1414

15-
public ParseTree Parse(SourceCodeFile sourceCodeFile)
15+
public ParseTree Parse(CodeFile sourceCodeFile)
1616
{
1717
ParseTree result = null;
1818

@@ -26,13 +26,16 @@ public ParseTree Parse(SourceCodeFile sourceCodeFile)
2626
AspxParseResult aspxTree = parser.Parse(source);
2727
foreach (var error in aspxTree.ParseErrors)
2828
{
29-
Logger.LogError(new ParsingException(filePath, message: error.Message) { TextSpan = error.Location.GetTextSpan() });
29+
Logger.LogError(new ParsingException(sourceCodeFile, message: error.Message)
30+
{
31+
TextSpan = error.Location.GetTextSpan()
32+
});
3033
}
3134
result = new AspxParseTree(aspxTree.RootNode);
3235
}
3336
catch (Exception ex)
3437
{
35-
Logger.LogError(new ParsingException(filePath, ex));
38+
Logger.LogError(new ParsingException(sourceCodeFile, ex));
3639
result = new CSharpRoslynParseTree();
3740
}
3841
}

0 commit comments

Comments
 (0)