Skip to content

Commit 6f69a00

Browse files
cmm-airbytecristina.mariscalmaxi297
authored
🎉 Source Salesforce: reduce info logs (#37340)
Co-authored-by: cristina.mariscal <[email protected]> Co-authored-by: Maxime Carbonneau-Leclerc <[email protected]>
1 parent 1f9dd51 commit 6f69a00

File tree

9 files changed

+74
-24
lines changed

9 files changed

+74
-24
lines changed

‎airbyte-integrations/connectors/source-salesforce/integration_tests/bulk_error_test.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ def test_failed_jobs_with_successful_switching(caplog, input_sandbox_config, str
8989
"id": "fake_id",
9090
},
9191
)
92-
m.register_uri("GET", job_matcher, json={"state": "Failed", "errorMessage": "unknown error"})
92+
m.register_uri("GET", job_matcher, json={"state": "Failed", "errorMessage": "unknown error", "id": "fake_id"})
9393
m.register_uri("DELETE", job_matcher, json={})
9494
with caplog.at_level(logging.WARNING):
9595
loaded_record_ids = set(

‎airbyte-integrations/connectors/source-salesforce/metadata.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ data:
1010
connectorSubtype: api
1111
connectorType: source
1212
definitionId: b117307c-14b6-41aa-9422-947e34922962
13-
dockerImageTag: 2.5.7
13+
dockerImageTag: 2.5.8
1414
dockerRepository: airbyte/source-salesforce
1515
documentationUrl: https://docs.airbyte.com/integrations/sources/salesforce
1616
githubIssueLabel: source-salesforce

‎airbyte-integrations/connectors/source-salesforce/pyproject.toml

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
33
build-backend = "poetry.core.masonry.api"
44

55
[tool.poetry]
6-
version = "2.5.7"
6+
version = "2.5.8"
77
name = "source-salesforce"
88
description = "Source implementation for Salesforce."
99
authors = [ "Airbyte <[email protected]>",]

‎airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py

+10-4
Original file line numberDiff line numberDiff line change
@@ -463,6 +463,9 @@ def wait_for_job(self, url: str) -> str:
463463
raise AirbyteTracedException(message=message, failure_type=FailureType.config_error, exception=error)
464464
else:
465465
raise error
466+
job_id = job_info["id"]
467+
if job_status != job_info["state"]:
468+
self.logger.info(f"Job {self.name}/{job_id} status changed from {job_status} to {job_info['state']}")
466469
job_status = job_info["state"]
467470
if job_status in ["JobComplete", "Aborted", "Failed"]:
468471
if job_status != "JobComplete":
@@ -471,17 +474,19 @@ def wait_for_job(self, url: str) -> str:
471474
if not error_message:
472475
# not all failed response can have "errorMessage" and we need to show full response body
473476
error_message = job_info
474-
self.logger.error(f"JobStatus: {job_status}, sobject options: {self.sobject_options}, error message: '{error_message}'")
475-
477+
self.logger.error(
478+
f"Job: {self.name}/{job_id}, JobStatus: {job_status}, sobject options: {self.sobject_options}, error message: '{error_message}'"
479+
)
480+
else:
481+
self.logger.info(f"Job: {self.name}/{job_id}, JobStatus: {job_status}")
476482
return job_status
477483

478484
if delay_timeout < self.MAX_CHECK_INTERVAL_SECONDS:
479485
delay_timeout = 0.5 + math.exp(delay_cnt) / 1000.0
480486
delay_cnt += 1
481487

482488
time.sleep(delay_timeout)
483-
job_id = job_info["id"]
484-
self.logger.info(
489+
self.logger.debug(
485490
f"Sleeping {delay_timeout} seconds while waiting for Job: {self.name}/{job_id} to complete. Current state: {job_status}"
486491
)
487492

@@ -508,6 +513,7 @@ def execute_job(self, query: str, url: str) -> Tuple[Optional[str], Optional[str
508513
if not job_id:
509514
return None, job_status
510515
job_full_url = f"{url}/{job_id}"
516+
self.logger.info(f"Job: {self.name}/{job_id} created, Job Full Url: {job_full_url}")
511517
job_status = self.wait_for_job(url=job_full_url)
512518
if job_status not in ["UploadComplete", "InProgress"]:
513519
break

‎airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py

+8-15
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
from airbyte_cdk.utils import AirbyteTracedException
3333
from conftest import encoding_symbols_parameters, generate_stream
3434
from requests.exceptions import ChunkedEncodingError, HTTPError
35+
from salesforce_job_response_builder import SalesforceJobResponseBuilder
3536
from source_salesforce.api import Salesforce
3637
from source_salesforce.exceptions import AUTHENTICATION_ERROR_MESSAGE_MAPPING
3738
from source_salesforce.source import SourceSalesforce
@@ -46,7 +47,7 @@
4647

4748
_A_CHUNKED_RESPONSE = [b"first chunk", b"second chunk"]
4849
_A_JSON_RESPONSE = {"id": "any id"}
49-
_A_SUCCESSFUL_JOB_CREATION_RESPONSE = {"state": "JobComplete"}
50+
_A_SUCCESSFUL_JOB_CREATION_RESPONSE = SalesforceJobResponseBuilder().with_state("JobComplete").get_response()
5051
_A_PK = "a_pk"
5152
_A_STREAM_NAME = "a_stream_name"
5253

@@ -182,7 +183,7 @@ def test_bulk_sync_pagination(stream_config, stream_api, requests_mock):
182183
stream: BulkIncrementalSalesforceStream = generate_stream("Account", stream_config, stream_api)
183184
job_id = "fake_job"
184185
requests_mock.register_uri("POST", stream.path(), json={"id": job_id})
185-
requests_mock.register_uri("GET", stream.path() + f"/{job_id}", json={"state": "JobComplete"})
186+
requests_mock.register_uri("GET", stream.path() + f"/{job_id}", json=SalesforceJobResponseBuilder().with_id(job_id).with_state("JobComplete").get_response())
186187
resp_text = ["Field1,LastModifiedDate,ID"] + [f"test,2021-11-16,{i}" for i in range(5)]
187188
result_uri = requests_mock.register_uri(
188189
"GET",
@@ -217,14 +218,6 @@ def _get_result_id(stream):
217218
return int(list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slices))[0]["ID"])
218219

219220

220-
def test_bulk_sync_successful(stream_config, stream_api):
221-
stream: BulkIncrementalSalesforceStream = generate_stream("Account", stream_config, stream_api)
222-
with requests_mock.Mocker() as m:
223-
job_id = _prepare_mock(m, stream)
224-
m.register_uri("GET", stream.path() + f"/{job_id}", [{"json": {"state": "JobComplete"}}])
225-
assert _get_result_id(stream) == 1
226-
227-
228221
def test_bulk_sync_successful_long_response(stream_config, stream_api):
229222
stream: BulkIncrementalSalesforceStream = generate_stream("Account", stream_config, stream_api)
230223
with requests_mock.Mocker() as m:
@@ -488,7 +481,7 @@ def test_given_retryable_error_when_download_data_then_retry(send_http_request_p
488481
@patch("source_salesforce.source.BulkSalesforceStream._non_retryable_send_http_request")
489482
def test_given_first_download_fail_when_download_data_then_retry_job_only_once(send_http_request_patch):
490483
sf_api = Mock()
491-
sf_api.generate_schema.return_value = {}
484+
sf_api.generate_schema.return_value = SalesforceJobResponseBuilder().with_state("JobComplete").get_response()
492485
sf_api.instance_url = "http://test_given_first_download_fail_when_download_data_then_retry_job.com"
493486
job_creation_return_values = [_A_JSON_RESPONSE, _A_SUCCESSFUL_JOB_CREATION_RESPONSE]
494487
send_http_request_patch.return_value.json.side_effect = job_creation_return_values * 2
@@ -876,13 +869,13 @@ def test_bulk_stream_request_params_states(stream_config_date_format, stream_api
876869
stream: BulkIncrementalSalesforceStream = generate_stream("Account", stream_config_date_format, stream_api, state=state, legacy=True)
877870

878871
job_id_1 = "fake_job_1"
879-
requests_mock.register_uri("GET", stream.path() + f"/{job_id_1}", [{"json": {"state": "JobComplete"}}])
872+
requests_mock.register_uri("GET", stream.path() + f"/{job_id_1}", [{"json": SalesforceJobResponseBuilder().with_id(job_id_1).with_state("JobComplete").get_response()}])
880873
requests_mock.register_uri("DELETE", stream.path() + f"/{job_id_1}")
881874
requests_mock.register_uri("GET", stream.path() + f"/{job_id_1}/results", text="Field1,LastModifiedDate,ID\ntest,2023-01-15,1")
882875
requests_mock.register_uri("PATCH", stream.path() + f"/{job_id_1}")
883876

884877
job_id_2 = "fake_job_2"
885-
requests_mock.register_uri("GET", stream.path() + f"/{job_id_2}", [{"json": {"state": "JobComplete"}}])
878+
requests_mock.register_uri("GET", stream.path() + f"/{job_id_2}", [{"json": SalesforceJobResponseBuilder().with_id(job_id_2).with_state("JobComplete").get_response()}])
886879
requests_mock.register_uri("DELETE", stream.path() + f"/{job_id_2}")
887880
requests_mock.register_uri(
888881
"GET", stream.path() + f"/{job_id_2}/results", text="Field1,LastModifiedDate,ID\ntest,2023-04-01,2\ntest,2023-02-20,22"
@@ -893,7 +886,7 @@ def test_bulk_stream_request_params_states(stream_config_date_format, stream_api
893886
queries_history = requests_mock.register_uri(
894887
"POST", stream.path(), [{"json": {"id": job_id_1}}, {"json": {"id": job_id_2}}, {"json": {"id": job_id_3}}]
895888
)
896-
requests_mock.register_uri("GET", stream.path() + f"/{job_id_3}", [{"json": {"state": "JobComplete"}}])
889+
requests_mock.register_uri("GET", stream.path() + f"/{job_id_3}", [{"json": SalesforceJobResponseBuilder().with_id(job_id_3).with_state("JobComplete").get_response()}])
897890
requests_mock.register_uri("DELETE", stream.path() + f"/{job_id_3}")
898891
requests_mock.register_uri("GET", stream.path() + f"/{job_id_3}/results", text="Field1,LastModifiedDate,ID\ntest,2023-04-01,3")
899892
requests_mock.register_uri("PATCH", stream.path() + f"/{job_id_3}")
@@ -952,7 +945,7 @@ def test_stream_slices_for_substream(stream_config, stream_api, requests_mock):
952945

953946
job_id = "fake_job"
954947
requests_mock.register_uri("POST", stream.path(), json={"id": job_id})
955-
requests_mock.register_uri("GET", stream.path() + f"/{job_id}", json={"state": "JobComplete"})
948+
requests_mock.register_uri("GET", stream.path() + f"/{job_id}", json=SalesforceJobResponseBuilder().with_id(job_id).with_state("JobComplete").get_response())
956949
requests_mock.register_uri(
957950
"GET",
958951
stream.path() + f"/{job_id}/results",

‎airbyte-integrations/connectors/source-salesforce/unit_tests/integration/test_bulk_stream.py

+3-2
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
from config_builder import ConfigBuilder
1313
from integration.utils import create_base_url, given_authentication, given_stream, read
1414
from salesforce_describe_response_builder import SalesforceDescribeResponseBuilder
15+
from salesforce_job_response_builder import SalesforceJobResponseBuilder
1516
from source_salesforce.streams import LOOKBACK_SECONDS
1617

1718
_A_FIELD_NAME = "a_field"
@@ -69,7 +70,7 @@ def test_when_read_then_create_job_and_extract_records_from_result(self) -> None
6970
)
7071
self._http_mocker.get(
7172
HttpRequest(f"{_BASE_URL}/jobs/query/{_JOB_ID}"),
72-
HttpResponse(json.dumps({"state": "JobComplete"})),
73+
SalesforceJobResponseBuilder().with_id(_JOB_ID).with_state("JobComplete").build(),
7374
)
7475
self._http_mocker.get(
7576
HttpRequest(f"{_BASE_URL}/jobs/query/{_JOB_ID}/results"),
@@ -118,7 +119,7 @@ def _create_sliced_job(self, start_date: datetime, first_upper_boundary: datetim
118119
)
119120
self._http_mocker.get(
120121
HttpRequest(f"{_BASE_URL}/jobs/query/{job_id}"),
121-
HttpResponse(json.dumps({"state": "JobComplete"})),
122+
SalesforceJobResponseBuilder().with_id(_JOB_ID).with_state("JobComplete").build(),
122123
)
123124
self._http_mocker.get(
124125
HttpRequest(f"{_BASE_URL}/jobs/query/{job_id}/results"),

‎airbyte-integrations/connectors/source-salesforce/unit_tests/resource/http/response/job_response.json

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
{
2+
"id": "750Tn000004ZoR3IAK",
3+
"operation": "queryAll",
4+
"object": "ActiveFeatureLicenseMetric",
5+
"createdById": "0050900000Bf63SAAR",
6+
"createdDate": "2024-04-25T15:50:37.000+0000",
7+
"systemModstamp": "2024-04-25T15:50:37.000+0000",
8+
"state": "UploadComplete",
9+
"concurrencyMode": "Parallel",
10+
"contentType": "CSV",
11+
"apiVersion": 57.0,
12+
"lineEnding": "LF",
13+
"columnDelimiter": "COMMA"
14+
}

‎airbyte-integrations/connectors/source-salesforce/unit_tests/salesforce_job_response_builder.py

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
2+
3+
import json
4+
5+
from airbyte_cdk.test.mock_http import HttpResponse
6+
from airbyte_cdk.test.mock_http.response_builder import HttpResponseBuilder, find_template
7+
8+
9+
class SalesforceJobResponseBuilder:
10+
def __init__(self):
11+
self._response = find_template("job_response", __file__)
12+
self._status_code = 200
13+
14+
def with_id(self, id: str) -> "HttpResponseBuilder":
15+
self._response["id"] = id
16+
return self
17+
18+
def with_state(self, state: str) -> "HttpResponseBuilder":
19+
self._response["state"] = state
20+
return self
21+
22+
def with_status_code(self, status_code: int) -> "HttpResponseBuilder":
23+
self._status_code = status_code
24+
return self
25+
26+
def with_error_message(self, error_message: int) -> "HttpResponseBuilder":
27+
self._response["errorMessage"] = error_message
28+
return self
29+
30+
def get_response(self) -> any:
31+
return self._response
32+
33+
def build(self) -> HttpResponse:
34+
return HttpResponse(json.dumps(self._response), self._status_code)
35+

‎docs/integrations/sources/salesforce.md

+1
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,7 @@ Now that you have set up the Salesforce source connector, check out the followin
193193

194194
| Version | Date | Pull Request | Subject |
195195
|:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------|
196+
| 2.5.8 | 2024-04-16 | [37340](https://github.com/airbytehq/airbyte/pull/37340) | Source Salesforce: reduce info logs |
196197
| 2.5.7 | 2024-04-24 | [36657](https://github.com/airbytehq/airbyte/pull/36657) | Schema descriptions |
197198
| 2.5.6 | 2024-04-19 | [37448](https://github.com/airbytehq/airbyte/pull/37448) | Ensure AirbyteTracedException in concurrent CDK are emitted with the right type |
198199
| 2.5.5 | 2024-04-18 | [37392](https://github.com/airbytehq/airbyte/pull/37419) | Ensure python return code != 0 in case of error |

0 commit comments

Comments
 (0)