S3Client::GetObject is not thread safe

[mikejiang]

Describe the bug

below is part of threat sanitizer alert ,

    #0 aws_cpu_has_feature <null> (libexternal_Saws_Slibaws-c-common.so+0x2023b)
    #1 aws_checksums_crc64nvme <null> (libexternal_Saws_Slibaws-checksums.so+0x14a6)
    #2 aws_checksums_crc64nvme_ex <null> (libexternal_Saws_Slibaws-checksums.so+0x1564)
    #3 Aws::Crt::Checksum::ComputeCRC64NVME(aws_byte_cursor, unsigned long) <null> (libexternal_Saws_Slibaws-crt-cpp.so+0x65691)
    #4 Aws::Utils::Crypto::CRCChecksum<unsigned long, &Aws::Crt::Checksum::ComputeCRC64NVME, &(Aws::Utils::Array<unsigned char> Aws::Utils::Crypto::ConvertToBuffer<unsigned long>(unsigned long))>::Update(unsigned char*, unsigned long) <null> (libexternal_Saws_Slibaws-sdk-core.so+0x31c3cc)
    #5 Aws::Utils::Crypto::CRC64::Update(unsigned char*, unsigned long) <null> (libexternal_Saws_Slibaws-sdk-core.so+0x306663)
    #6 WriteData(char*, unsigned long, unsigned long, void*) <null> (libexternal_Saws_Slibaws-sdk-core.so+0x27be9b)
    #7 Curl_client_write <null> (libexternal_Scurl_Slibcurl.so+0x6b47b)
    #8 Curl_readwrite <null> (libexternal_Scurl_Slibcurl.so+0x8721a)
    #9 multi_runsingle <null> (libexternal_Scurl_Slibcurl.so+0x5f675)
    #10 curl_multi_perform <null> (libexternal_Scurl_Slibcurl.so+0x61b19)
    #11 curl_easy_perform <null> (libexternal_Scurl_Slibcurl.so+0x3328a)
    #12 Aws::Http::CurlHttpClient::MakeRequest(std::shared_ptr<Aws::Http::HttpRequest> const&, Aws::Utils::RateLimits::RateLimiterInterface*, Aws::Utils::RateLimits::RateLimiterInterface*) const <null> (libexternal_Saws_Slibaws-sdk-core.so+0x283d08)
    #13 std::_Function_handler<std::shared_ptr<Aws::Http::HttpResponse> (), Aws::Client::AWSClient::AttemptOneRequest(std::shared_ptr<Aws::Http::HttpRequest> const&, Aws::AmazonWebServiceRequest const&, char const*, char const*, char const*) const::{lambda()#3}>::_M_invoke(std::_Any_data const&) <null> (libexternal_Saws_Slibaws-sdk-core.so+0x156688)
    #14 std::shared_ptr<Aws::Http::HttpResponse> smithy::components::tracing::TracingUtils::MakeCallWithTiming<std::shared_ptr<Aws::Http::HttpResponse> >(std::function<std::shared_ptr<Aws::Http::HttpResponse> ()>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, smithy::components::tracing::Meter const&, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >&&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) <null> (libexternal_Saws_Slibaws-sdk-core.so+0x17c886)
    #15 Aws::Client::AWSClient::AttemptOneRequest(std::shared_ptr<Aws::Http::HttpRequest> const&, Aws::AmazonWebServiceRequest const&, char const*, char const*, char const*) const <null> (libexternal_Saws_Slibaws-sdk-core.so+0x1615d1)
    #16 Aws::Client::AWSClient::AttemptExhaustively(Aws::Http::URI const&, Aws::AmazonWebServiceRequest const&, Aws::Http::HttpMethod, char const*, char const*, char const*) const <null> (libexternal_Saws_Slibaws-sdk-core.so+0x1644b3)
    #17 Aws::Client::AWSClient::MakeRequestWithUnparsedResponse(Aws::Http::URI const&, Aws::AmazonWebServiceRequest const&, Aws::Http::HttpMethod, char const*, char const*, char const*) const <null> (libexternal_Saws_Slibaws-sdk-core.so+0x1695d5)
    #18 Aws::Client::AWSClient::MakeRequestWithUnparsedResponse(Aws::AmazonWebServiceRequest const&, Aws::Endpoint::AWSEndpoint const&, Aws::Http::HttpMethod, char const*, char const*, char const*) const <null> (libexternal_Saws_Slibaws-sdk-core.so+0x16a1a9)
    #19 Aws::S3::S3Client::GetObject(Aws::S3::Model::GetObjectRequest const&) const::{lambda()#1}::opera r()() const <null> (libexternal_Saws_Slibaws.so+0x8256f3)
    #20 std::_Function_handler<Aws::Utils::Outcome<Aws::S3::Model::GetObjectResult, Aws::S3::S3Error> (), Aws::S3::S3Client::GetObject(Aws::S3::Model::GetObjectRequest const&) const::{lambda()#1}>::_M_invoke(std::_Any_data const&) <null> (libexternal_Saws_Slibaws.so+0x825c8c)
    #21 Aws::S3::S3Client::GetObject(Aws::S3::Model::GetObjectRequest const&) const <null> (libexternal_Saws_Slibaws.so+0x7793b0)
    #22 Aws::Transfer::TransferManager::DoSinglePartDownload(std::shared_ptr<Aws::Transfer::TransferHandle> const&) <null> (libexternal_Saws_Slibaws.so+0xbc0fc3)
    #23 Aws::Transfer::TransferManager::DoDownload(std::shared_ptr<Aws::Transfer::TransferHandle> const&) <null> (libexternal_Saws_Slibaws.so+0xbc2d5e)
    #24 std::_Function_handler<void (), std::_Bind<Aws::Transfer::TransferManager::DownloadFile(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::function<std::iostream* ()>, Aws::Transfer::DownloadConfiguration const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::shared_ptr<Aws::Client::AsyncCallerContext const> const&)::{lambda()#1} ()> >::_M_invoke(std::_Any_data const&) <null> (libexternal_Saws_Slibaws.so+0xbc4ec8)
    #25 Aws::Utils::Threading::ThreadTask::MainTaskRunner() <null> (libexternal_Saws_Slibaws-sdk-core.so+0x3413c5)
    #26 std::thread::_State_impl<std::thread::_Invoker<std::tuple<std::_Bi

Aws::S3::S3Client::GetObject call will hit Aws::Crt::Checksum::ComputeCRC64NVME , which mutates some static function pointers and variables, causing racing.
I didn't find any reference in terms of how to properly initialize these checkssum related global state.
had to do the hack by computing it ahead of time in my code

Aws::Utils::Crypto::CRC32 crc32;
  Aws::Utils::Crypto::CRC32C crc32c;
  Aws::Utils::Crypto::CRC64 crc64;

  Aws::String dummy = "dummy";
  crc32.Calculate(dummy);
  crc32c.Calculate(dummy);
  crc64.Calculate(dummy);

so that these statics get set before kicks off the concurrent download logic.

what is the kosher way of doing this?

Regression Issue

• Select this option if this issue appears to be a regression.

Expected Behavior

these gets set during Aws::InitAPI

Current Behavior

they are not set until getObject() is invoked

Reproduction Steps

 Aws::InitAPI();
 Aws::Client::ClientConfiguration conf;
  conf.region = aws_region;
  Aws::S3::S3Client client(conf);

auto executor = ozette::Executor::Create(3);
  executor->ParallelFor(0, 3, [&](int64_t i) {
  Aws::Http::URI uri(s3_uri.c_str());
  Aws::S3::Model::GetObjectRequest get_request;
  get_request.SetBucket(uri.GetAuthority());
  get_request.SetKey(uri.GetPath());
  auto get_object_outcome = s3_client.GetObject(get_request);

  });

Possible Solution

No response

Additional Information/Context

No response

AWS CPP SDK version used

1.11.504

Compiler and Version used

gcc9

Operating System and version

ubuntu

[SergeyRyabinin]

Hi!

Thank you for the report! We have reviewed it, and CRT library is going to fix it sometime.
In the meanwhile, we can confirm this is somewhat a false-positive report: this is a side-effect of lazy selection of performance optimized method for the checksum computation.
This init happens on the very first call, so if there are multiple concurrent first calls to the checksum computation - it results in the reported race condition.
However, there must be no real memory/data corruption, or any race condition side-effects, as the result of concurrent optimized checksum method selection will be the same on all invocations.

I submitted an internal tracking ticket on the CRT library for this report.

Best regards,
Sergey

[DmitriyMusatkin]

should be fixed in 1.11.536

[mikejiang]

upgraded sdk, so after adding this to my code prior to invoke any other functions

static struct aws_allocator allocator_;
aws_checksums_library_init(&allocator_);

still get data race error

  Previous write of size 1 at 0x7ffff41a32e1 by thread T2:
    #0 aws_checksums_crc32c_clmul_sse42 <null> (libexternal_Saws_Slibaws-checksums.so+0x2d28)
    #1 aws_checksums_crc32c_intel_avx512_with_sse_fallback <null> (libexternal_Saws_Slibaws-checksums.so+0x2ef7)
    #2 aws_checksums_crc32c <null> (libexternal_Saws_Slibaws-checksums.so+0x1726)
    #3 aws_checksums_crc32c_ex <null> (libexternal_Saws_Slibaws-checksums.so+0x17c3)
    #4 Aws::Crt::Checksum::ComputeCRC32C(aws_byte_cursor, unsigned int) <null> (libexternal_Saws_Slibaws-crt-cpp.so+0x65811)
    #5 Aws::Utils::Crypto::CRCChecksum<unsigned int, &Aws::Crt::Checksum::ComputeCRC32C, &(Aws::Utils::Array<unsigned char> Aws::Utils::Crypto::ConvertToBuffer<unsigned int>(unsigned int))>::Update(unsigned char*, unsigned long) <null> (libexternal_Saws_Slibaws-sdk-core.so+0x32986b)
    #6 Aws::Utils::Crypto::CRC32C::Update(unsigned char*, unsigned long) <null> (libexternal_Saws_Slibaws-sdk-core.so+0x31334e)
    #7 WriteData(char*, unsigned long, unsigned long, void*) <null> (libexternal_Saws_Slibaws-sdk-core.so+0x282c0b)
    #8 Curl_client_write <null> (libexternal_Scurl_Slibcurl.so+0x6b47b)
    #9 Curl_readwrite <null> (libexternal_Scurl_Slibcurl.so+0x8721a)
    #10 multi_runsingle <null> (libexternal_Scurl_Slibcurl.so+0x5f675)
    #11 curl_multi_perform <null> (libexternal_Scurl_Slibcurl.so+0x61b19)
    #12 curl_easy_perform <null> (libexternal_Scurl_Slibcurl.so+0x3328a)
    #13 Aws::Http::CurlHttpClient::MakeRequest(std::shared_ptr<Aws::Http::HttpRequest> const&, Aws::Utils::RateLimits::RateLimiterInterface*, Aws::Utils::RateLimits::RateLimiterInterface*) const <null> (libexternal_Saws_Slibaws-sdk-core.so+0x28ab2d)
    #14 std::_Function_handler<std::shared_ptr<Aws::Http::HttpResponse> (), Aws::Client::AWSClient::AttemptOneRequest(std::shared_ptr<Aws::Http::HttpRequest> const&, Aws::AmazonWebServiceRequest const&, char const*, char const*, char const*) const::{lambda()#3}>::_M_invoke(std::_Any_data const&) <null> (libexternal_Saws_Slibaws-sdk-core.so+0x15b9b8)
    #15 std::shared_ptr<Aws::Http::HttpResponse> smithy::components::tracing::TracingUtils::MakeCallWithTiming<std::shared_ptr<Aws::Http::HttpResponse> >(std::function<std::shared_ptr<Aws::Http::HttpResponse> ()>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, smithy::components::tracing::Meter const&, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >&&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) <null> (libexternal_Saws_Slibaws-sdk-core.so+0x181e96)
    #16 Aws::Client::AWSClient::AttemptOneRequest(std::shared_ptr<Aws::Http::HttpRequest> const&, Aws::AmazonWebServiceRequest const&, char const*, char const*, char const*) const <null> (libexternal_Saws_Slibaws-sdk-core.so+0x166a44)
    #17 Aws::Client::AWSClient::AttemptExhaustively(Aws::Http::URI const&, Aws::AmazonWebServiceRequest const&, Aws::Http::HttpMethod, char const*, char const*, char const*) const <null> (libexternal_Saws_Slibaws-sdk-core.so+0x169943)
    #18 Aws::Client::AWSClient::MakeRequestWithUnparsedResponse(Aws::Http::URI const&, Aws::AmazonWebServiceRequest const&, Aws::Http::HttpMethod, char const*, char const*, char const*) const <null> (libexternal_Saws_Slibaws-sdk-core.so+0x16ebf5)
    #19 Aws::Client::AWSClient::MakeRequestWithUnparsedResponse(Aws::AmazonWebServiceRequest const&, Aws::Endpoint::AWSEndpoint const&, Aws::Http::HttpMethod, char const*, char const*, char const*) const <null> (libexternal_Saws_Slibaws-sdk-core.so+0x16f7c9)
    #20 Aws::S3::S3Client::GetObject(Aws::S3::Model::GetObjectRequest const&) const::{lambda()#1}::operator()() const <null> (libexternal_Saws_Slibaws.so+0x842533)
    #21 std::_Function_handler<Aws::Utils::Outcome<Aws::S3::Model::GetObjectResult, Aws::S3::S3Error> (), Aws::S3::S3Client::GetObject(Aws::S3::Model::GetObjectRequest const&) const::{lambda()#1}>::_M_invoke(std::_Any_data const&) <null> (libexternal_Saws_Slibaws.so+0x842acc)

is it because aws_checksums_crc32c still not initialized?

[DmitriyMusatkin]

just to confirm you updated to 1.11.536 and updated submodules and did a whole rebuild?
fix is in crt and all checksums are now initialized when crt loads