fix: speed up NonAsciiExtractionRule

terales · terales · commit b39150ba5e62 · 2021-09-12T21:11:30.000+03:00
diff --git a/src/extraction/rules/non-ascii-characters.ts b/src/extraction/rules/non-ascii-characters.ts
@@ -6,9 +6,9 @@ export class NonAsciiExtractionRule extends ExtractionRule {
   shouldExtract(str: string) {
     // [^\u{0}-\u{7F}] -- non Latin script,see https://unicode.org/reports/tr18/#General_Category_Property
     const words = str.match(/\p{Letter}*/ug) ?? []
-    const wordsWithNonAsciiChars = words.filter(word => word.match(/[^\u{0}-\u{7F}]/u))
+    const containsWordWithNonAsciiChar = words.some(word => word.match(/[^\u{0}-\u{7F}]/u))
 
-    if (wordsWithNonAsciiChars.length > 0)
+    if (containsWordWithNonAsciiChar)
       return ExtractionScore.MustInclude
   }
 }

Original file line number	Diff line number	Diff line change
`@@ -6,9 +6,9 @@ export class NonAsciiExtractionRule extends ExtractionRule {`
`6`	`6`	`shouldExtract(str: string) {`
`7`	`7`	`// [^\u{0}-\u{7F}] -- non Latin script,see https://unicode.org/reports/tr18/#General_Category_Property`
`8`	`8`	`const words = str.match(/\p{Letter}*/ug) ?? []`
`9`		`- const wordsWithNonAsciiChars = words.filter(word => word.match(/[^\u{0}-\u{7F}]/u))`
	`9`	`+ const containsWordWithNonAsciiChar = words.some(word => word.match(/[^\u{0}-\u{7F}]/u))`
`10`	`10`
`11`		`- if (wordsWithNonAsciiChars.length > 0)`
	`11`	`+ if (containsWordWithNonAsciiChar)`
`12`	`12`	`return ExtractionScore.MustInclude`
`13`	`13`	`}`
`14`	`14`	`}`