Syncing recent changes. (#1088)

* Syncing recent changes.

Notable changes:

* Parsers support fully removed from the artifacts collector.
* Interrogate is no longer artifact-collector based.
* RDFProtoStruct -> plain protos migration in progress.
* Text type no longer used (replaced with 'str').

Author: mbushkov

CHANGELOG.md

@@ -27,6 +27,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   https://github.com/google/grr/pkgs/container/grr
 * Docker compose configuration file to run all GRR/Fleetspeak components in
   separate Docker containers.
+* Python API was extended by a function (`DecodeCrowdStrikeQuarantineEncoding`)
+  to decode a crowdstrike quarantine encoded file, given as a
+  `BinaryChunkIterator`.
 
 
 ### API removed
@@ -45,6 +48,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   OSReleaseBreakdown7ReportPlugin, OSReleaseBreakdown14ReportPlugin,
   OSReleaseBreakdown30ReportPlugin, SystemFlowsReportPlugin,
   UserFlowsReportPlugin, MostActiveUsersReportPlugin, UserActivityReportPlugin).
+* GetFileDecoders API method
+  (`/api/clients/<client_id>/vfs-decoders/<path:file_path>`). Getting file
+  decoders functionality was removed as it was not used before.
+* GetDecodedFileBlob API method (`/api/clients/<client_id>/vfs-decoded-blob/`).
+  Get decoded file blob functionality was removed as it was unused before. Only
+  one decoder for decoding crowdstrike quarantine encoded files was implemented,
+  this functionality is now exposed via the Python API.
 
 ### Planned for removal
 

api_client/python/README.md

@@ -194,6 +194,14 @@ grr_api_shell --basic_auth_username "user" --basic_auth_password "pwd" \
   http://localhost:1234
 ```
 
+Decode a Crowdstrike encoded file that was fetched via GetBlob:
+
+```bash
+grr_api_shell --basic_auth_username "user" --basic_auth_password "pwd" \
+  --exec_code 'grrapi.Client("C.1234567890ABCDEF").File("/fs/os/var/encrypted/file").GetBlob().DecodeCrowdStrikeQuarantineEncoding().WriteToFile("./decoded.file")' \
+  http://localhost:1234
+```
+
 
 Download an archive of all files collected with OS-handler (not TSK/NTFS) from a
 GRR client:

api_client/python/grr_api_client/api.py

@@ -4,7 +4,6 @@
 from typing import Any
 from typing import Dict
 from typing import Optional
-from typing import Text
 from typing import Tuple
 
 from google.protobuf import message
@@ -95,7 +94,7 @@ def GrrBinary(
   def GrrUser(self) -> user.GrrUser:
     return user.GrrUser(context=self._context)
 
-  def UploadYaraSignature(self, signature: Text) -> bytes:
+  def UploadYaraSignature(self, signature: str) -> bytes:
     """Uploads the specified YARA signature.
 
     Args:

api_client/python/grr_api_client/hunt.py

@@ -449,8 +449,7 @@ def CreateHunt(
 
   request = hunt_pb2.ApiCreateHuntArgs(flow_name=flow_name)
   if flow_args:
-    request.flow_args.value = flow_args.SerializeToString()
-    request.flow_args.type_url = utils.GetTypeUrl(flow_args)
+    request.flow_args.Pack(flow_args)
 
   if hunt_runner_args:
     request.hunt_runner_args.CopyFrom(hunt_runner_args)

api_client/python/grr_api_client/utils.py

@@ -84,13 +84,15 @@ def MapItemsIterator(
   )
 
 
-class BinaryChunkIterator(object):
+class BinaryChunkIterator:
   """Iterator object for binary streams."""
 
+  chunks: Iterator[bytes]
+
   def __init__(self, chunks: Iterator[bytes]) -> None:
     super().__init__()
 
-    self.chunks = chunks  # type: Iterator[bytes]
+    self.chunks = chunks
 
   def __iter__(self) -> Iterator[bytes]:
     for c in self.chunks:
@@ -107,6 +109,32 @@ def WriteToFile(self, file_name: str) -> None:
     with open(file_name, "wb") as fd:
       self.WriteToStream(fd)
 
+  def DecodeCrowdStrikeQuarantineEncoding(self) -> "BinaryChunkIterator":
+    """Decodes Crowdstrike quarantine file."""
+
+    def DecoderGenerator() -> Iterator[bytes]:
+      for index, chunk in enumerate(self):
+        if index == 0:
+          if len(chunk) < 12:
+            raise ValueError(
+                "Unsupported chunk size, chunks need to be at least 12 bytes"
+            )
+
+          if chunk[0:4] != b"CSQD":
+            raise ValueError(
+                "File does not start with Crowdstrike quarantine identifier"
+            )
+
+          # TODO: Add a check if the actual file size matches the
+          # value in chunk[4:12].
+
+          # The remainder of the first chunk belongs to the actual file.
+          chunk = chunk[12:]
+
+        yield Xor(chunk, 0x7E)
+
+    return BinaryChunkIterator(DecoderGenerator())
+
 
 # Default poll interval in seconds.
 DEFAULT_POLL_INTERVAL: int = 15
@@ -274,6 +302,11 @@ def Recurse(msg: message.Message, prev: Tuple[str, ...]) -> None:
   return result
 
 
+def Xor(bytestr: bytes, key: int) -> bytes:
+  """Returns a `bytes` object where each byte has been xored with key."""
+  return bytes([byte ^ key for byte in bytestr])
+
+
 def RegisterProtoDescriptors(
     db: symbol_database.SymbolDatabase,
     *additional_descriptors: descriptor.FileDescriptor,

api_client/python/grr_api_client/utils_test.py

@@ -1,4 +1,7 @@
 #!/usr/bin/env python
+import io
+import struct
+
 from absl.testing import absltest
 
 from google.protobuf import empty_pb2
@@ -53,5 +56,45 @@ def testTransform(self):
     self.assertEqual(dct, {"value": 1337 * 2})
 
 
+class GetCrowdstrikeDecodedBlobTest(absltest.TestCase):
+
+  def _CrowdstrikeEncode(self, data) -> bytes:
+    buf = io.BytesIO()
+    buf.write(b"CSQD")
+    buf.write(struct.pack("<Q", len(data)))
+    buf.write(utils.Xor(data, 0x7E))
+    return buf.getvalue()
+
+  def testDecodeSingleChunk(self):
+    content = b"foobarbaz"
+    iterator_content = iter((self._CrowdstrikeEncode(content),))
+    encoded_content = utils.BinaryChunkIterator(iterator_content)
+
+    decoded = encoded_content.DecodeCrowdStrikeQuarantineEncoding()
+    self.assertIsInstance(decoded, utils.BinaryChunkIterator)
+    self.assertEqual(b"".join(decoded), content)
+
+  def testDecode_RaisesIfCrowdstrikeIdentifierBytesMissing(self):
+    content = b"foobarbazbang"
+    iterator_content = iter((content,))
+    encoded_content = utils.BinaryChunkIterator(iterator_content)
+
+    with self.assertRaises(ValueError):
+      list(encoded_content.DecodeCrowdStrikeQuarantineEncoding())
+
+  def testDecodeSeveralChunks(self):
+    content = b"ABC" * 1024
+    encoded_content = self._CrowdstrikeEncode(content)
+    first_chunk = encoded_content[0:1024]
+    second_chunk = encoded_content[1024:2048]
+    third_chunk = encoded_content[2048:]
+    encoded_content = utils.BinaryChunkIterator(
+        iter((first_chunk, second_chunk, third_chunk))
+    )
+
+    decoded = encoded_content.DecodeCrowdStrikeQuarantineEncoding()
+    self.assertEqual(b"".join(decoded), content)
+
+
 if __name__ == "__main__":
   absltest.main()

api_client/python/grr_api_client/yara.py

@@ -1,14 +1,12 @@
 #!/usr/bin/env python
 """A module with YARA convenience wrappers for the GRR API."""
 
-from typing import Text
-
 from grr_api_client import context as api_context
 from grr_response_proto.api import yara_pb2
 
 
 def UploadYaraSignature(
-    signature: Text,
+    signature: str,
     context: api_context.GrrApiContext,
 ) -> bytes:
   """Uploads the specified YARA signature.