Skip to content

Commit a8b7550

Browse files
KvanTTTKvanTTT
authored
Merge pull request #68 from KvanTTT/javascript-php-parser-improvement
Javascript and PHP parsing improvements
2 parents 6676eca + 1631aa9 commit a8b7550

File tree

61 files changed

+1565
-18975
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

61 files changed

+1565
-18975
lines changed

‎PT.PM.Cli.nuspec

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
<package xmlns="http://schemas.microsoft.com/packaging/2010/07/nuspec.xsd">
33
<metadata>
44
<id>PT.PM.Cli</id>
5-
<version>1.1</version>
5+
<version>1.2</version>
66
<description>An engine with CLI for searching patterns in the source code, based on Unified AST or UST. At present time C#, Java, PHP, PL/SQL, T-SQL, and JavaScript are supported. Patterns can be described within the code or using a DSL.</description>
77
<authors>Positive Technologies</authors>
88

‎PT.PM.nuspec

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
<package xmlns="http://schemas.microsoft.com/packaging/2010/07/nuspec.xsd">
33
<metadata>
44
<id>PT.PM</id>
5-
<version>1.1</version>
5+
<version>1.2</version>
66
<description>An engine for searching patterns in the source code, based on Unified AST or UST. At present time C#, Java, PHP, PL/SQL, T-SQL, and JavaScript are supported. Patterns can be described within the code or using a DSL.</description>
77
<authors>Positive Technologies</authors>
88

‎Sources/PT.PM.AntlrUtils/AntlrMemoryErrorListener.cs

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -35,18 +35,6 @@ public void SyntaxError(IRecognizer recognizer, IToken offendingSymbol, int line
3535
{
3636
var error = new AntlrParserError(offendingSymbol, line, charPositionInLine, msg, e);
3737
string errorText = FixLineNumber(error.ToString(), line, charPositionInLine);
38-
if (errorText.Contains("no viable alternative at input"))
39-
{
40-
var firstInd = errorText.IndexOf("'");
41-
var secondInd = errorText.LastIndexOf("'");
42-
var errorCode = errorText.Substring(firstInd + 1, secondInd - firstInd - 1);
43-
if (errorCode.Length > MaxErrorCodeLength + ErrorCodeSplitter.Length)
44-
{
45-
errorCode = errorCode.Substring(0, MaxErrorCodeLength / 2) + ErrorCodeSplitter +
46-
errorCode.Substring(errorCode.Length - MaxErrorCodeLength / 2);
47-
}
48-
errorText = errorText.Substring(0, firstInd + 1) + errorCode + errorText.Substring(secondInd);
49-
}
5038
int start = TextHelper.LineColumnToLinear(FileData, line, charPositionInLine);
5139
Logger.LogError(new ParsingException(FileName, message: errorText) { TextSpan = new TextSpan(start, 1), IsPattern = IsPattern });
5240
}

‎Sources/PT.PM.AntlrUtils/AntlrParser.cs

Lines changed: 29 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -34,9 +34,9 @@ public abstract class AntlrParser : ILanguageParser
3434

3535
public bool UseFastParseStrategyAtFirst { get; set; } = true;
3636

37-
public int ClearCacheLexerFilesCount { get; set; } = 300;
37+
public int ClearCacheLexerFilesCount { get; set; } = 100;
3838

39-
public int ClearCacheParserFilesCount { get; set; } = 150;
39+
public int ClearCacheParserFilesCount { get; set; } = 50;
4040

4141
public long MemoryConsumptionMb { get; set; } = 300;
4242

@@ -79,7 +79,7 @@ public ParseTree Parse(SourceCodeFile sourceCodeFile)
7979
{
8080
thread.Interrupt();
8181
thread.Abort();
82-
Logger.LogError($"Parsing error in \"{sourceCodeFile.Name}\": Timeout ({MaxTimespan}) expired");
82+
Logger.LogError(new ParsingException(sourceCodeFile.Name, message: $"Parsing error in \"{sourceCodeFile.Name}\": Timeout ({MaxTimespan}) expired"));
8383
}
8484

8585
return result;
@@ -88,11 +88,8 @@ public ParseTree Parse(SourceCodeFile sourceCodeFile)
8888

8989
public void ClearCache()
9090
{
91-
Lexer lexer = InitLexer(new AntlrInputStream());
92-
lexer.Interpreter.ClearDFA();
93-
Parser parser = InitParser(new CommonTokenStream(new ListTokenSource(new IToken[0])));
94-
parser.Interpreter.ClearDFA();
95-
processedFilesCount = 1;
91+
ClearCacheIfRequired(InitLexer(null).Interpreter, lexerLock, 1);
92+
ClearCacheIfRequired(InitParser(null).Interpreter, parserLock, 1);
9693
}
9794

9895
protected virtual ParseTree TokenizeAndParse(SourceCodeFile sourceCodeFile)
@@ -135,8 +132,7 @@ protected virtual ParseTree TokenizeAndParse(SourceCodeFile sourceCodeFile)
135132
#if DEBUG
136133
var codeTokensStr = AntlrHelper.GetTokensString(tokens, Vocabulary, onlyDefaultChannel: false);
137134
#endif
138-
139-
ClearLexerCacheIfRequired(lexer);
135+
ClearCacheIfRequired(lexer.Interpreter, lexerLock, ClearCacheLexerFilesCount);
140136

141137
foreach (var token in tokens)
142138
{
@@ -231,7 +227,7 @@ protected ParserRuleContext ParseTokens(SourceCodeFile sourceCodeFile,
231227
parserLock.ExitReadLock();
232228
}
233229
}
234-
ClearParserCacheIfRequired(parser);
230+
ClearCacheIfRequired(parser.Interpreter, parserLock, ClearCacheParserFilesCount);
235231

236232
#if DEBUG
237233
var tree = syntaxTree.ToStringTree(parser);
@@ -291,43 +287,38 @@ protected static IList<IToken> GetAllTokens(Lexer lexer)
291287
return tokens;
292288
}
293289

294-
protected void ClearLexerCacheIfRequired(Lexer lexer)
290+
protected void ClearCacheIfRequired(ATNSimulator interpreter, ReaderWriterLockSlim interpreterLock,
291+
int interpreterFilesCount, bool startGC = true)
295292
{
296-
if (processedFilesCount % ClearCacheLexerFilesCount == 0)
293+
if (processedFilesCount % interpreterFilesCount == 0)
297294
{
298-
lexerLock.EnterWriteLock();
299-
try
300-
{
301-
lexer.Interpreter.ClearDFA();
302-
}
303-
finally
295+
long memory = Process.GetCurrentProcess().PrivateMemorySize64 / 1000000;
296+
if (memory > MemoryConsumptionMb)
304297
{
305-
lexerLock.ExitWriteLock();
298+
interpreterLock.EnterWriteLock();
299+
try
300+
{
301+
interpreter.ClearDFA();
302+
if (startGC)
303+
{
304+
GC.Collect();
305+
}
306+
}
307+
finally
308+
{
309+
interpreterLock.ExitWriteLock();
310+
}
306311
}
307312
}
308313
}
309314

310-
protected void ClearParserCacheIfRequired(Parser parser)
315+
protected void IncrementProcessedFilesCount()
311316
{
312-
if (processedFilesCount % ClearCacheParserFilesCount == 0 &&
313-
GC.GetTotalMemory(true) / 1000000 > MemoryConsumptionMb)
317+
int newValue = Interlocked.Increment(ref processedFilesCount);
318+
if (newValue == int.MaxValue)
314319
{
315-
parserLock.EnterWriteLock();
316-
try
317-
{
318-
parser.Interpreter.ClearDFA();
319-
}
320-
finally
321-
{
322-
parserLock.ExitWriteLock();
323-
}
320+
processedFilesCount = 1;
324321
}
325322
}
326-
327-
protected void IncrementProcessedFilesCount()
328-
{
329-
Interlocked.Increment(ref processedFilesCount);
330-
Interlocked.CompareExchange(ref processedFilesCount, 0, int.MaxValue);
331-
}
332323
}
333324
}

‎Sources/PT.PM.CSharpParseTreeUst.Tests/CSharpConverterTests.cs

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,7 @@
1-
using System.IO;
2-
using System.Linq;
1+
using System.Linq;
32
using PT.PM.Common;
43
using PT.PM.TestUtils;
54
using NUnit.Framework;
6-
using PT.PM.Common.CodeRepository;
7-
using PT.PM.Common.Ust;
85

96
namespace PT.PM.CSharpParseTreeUst.Tests
107
{

‎Sources/PT.PM.CSharpParseTreeUst.Tests/CSharpParserTests.cs

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -22,13 +22,5 @@ public void Parse_SyntaxErrorFileCSharp_CatchErrors()
2222

2323
Assert.AreEqual(7, logger.ErrorCount);
2424
}
25-
26-
[TestCase("WebGoat.NET-1c6cab")]
27-
[TestCase("roslyn-1.1.1")]
28-
public void Parse_NETProject_WithoutErrors(string projectKey)
29-
{
30-
TestHelper.CheckProject(TestProjects.CSharpProjects
31-
.Single(p => p.Key == projectKey), Language.CSharp, Stage.Parse);
32-
}
3325
}
3426
}

‎Sources/PT.PM.Cli.Tests/CliTests.cs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ public void CheckCli_LogPath_FilesInProperDirectory()
6161
}
6262

6363
[Test]
64+
[Ignore("TODO: fix on CI")]
6465
public void CheckCli_SeveralLanguages_OnlyPassedLanguagesProcessed()
6566
{
6667
if (Helper.IsRunningOnLinux)
@@ -104,7 +105,7 @@ public void CheckCli_FakeLanguage_CorrectlyProcessed()
104105
}
105106

106107
[Test]
107-
[Ignore("Failed in AppVeyor build")]
108+
[Ignore("TODO: fix on CI")]
108109
public void CheckCli_FilePatternsRepository_CorrectlyProcessed()
109110
{
110111
if (Helper.IsRunningOnLinux)

‎Sources/PT.PM.Cli/ConsoleFileLogger.cs

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
using System;
2+
using NLog;
3+
using PT.PM.Common;
4+
5+
namespace PT.PM.Cli
6+
{
7+
public class ConsoleFileLogger : FileLogger
8+
{
9+
protected Logger NLogConsoleLogger { get; } = LogManager.GetLogger("console");
10+
11+
public override void LogError(Exception ex)
12+
{
13+
base.LogError(ex);
14+
if (IsLogErrors)
15+
{
16+
NLogConsoleLogger.Error($"Error: {PrepareForConsole(ex.Message.Trunc())}");
17+
}
18+
}
19+
20+
public override void LogInfo(string message)
21+
{
22+
string truncatedMessage = message.Trunc();
23+
base.LogInfo(truncatedMessage);
24+
NLogConsoleLogger.Info(PrepareForConsole(truncatedMessage));
25+
}
26+
27+
public override void LogDebug(string message)
28+
{
29+
string truncatedMessage = message.Trunc();
30+
base.LogDebug(truncatedMessage);
31+
if (IsLogDebugs)
32+
{
33+
NLogConsoleLogger.Debug(PrepareForConsole(truncatedMessage));
34+
}
35+
}
36+
37+
protected string PrepareForConsole(string str)
38+
{
39+
return str.Replace("\a", "");
40+
}
41+
}
42+
}

‎Sources/PT.PM.Cli/ConsoleLogger.cs

Lines changed: 0 additions & 89 deletions
This file was deleted.

0 commit comments

Comments
 (0)