Closed
Description
Environment
- Airbyte version: latest
- OS Version / Instance: using helm charts on digital ocean
- Deployment: Kubernetes
- Source Connector and version: any
- Severity: Critical
- Step where error happened: Setup new connection
Current Behavior
2022-03-22 18:05:12 �[32mINFO�[m i.a.w.p.KubeProcessFactory(create):113 - Attempting to start pod = source-github-sync-ea7da5c8-55d6-4d4d-a435-656586c8f876-0-ncbss
2022-03-22 18:05:12 �[32mINFO�[m i.a.w.p.KubeProcessFactory(create):116 - source-github-sync-ea7da5c8-55d6-4d4d-a435-656586c8f876-0-ncbss stdoutLocalPort = 9006
2022-03-22 18:05:12 �[32mINFO�[m i.a.w.p.KubeProcessFactory(create):119 - source-github-sync-ea7da5c8-55d6-4d4d-a435-656586c8f876-0-ncbss stderrLocalPort = 9007
2022-03-22 18:05:12 �[32mINFO�[m i.a.w.p.KubePodProcess(lambda$setupStdOutAndStdErrListeners$8):550 - Creating stdout socket server...
2022-03-22 18:05:12 �[32mINFO�[m i.a.w.p.KubePodProcess(<init>):507 - Creating pod...
2022-03-22 18:05:12 �[32mINFO�[m i.a.w.p.KubePodProcess(lambda$setupStdOutAndStdErrListeners$9):568 - Creating stderr socket server...
2022-03-22 18:05:12 �[32mINFO�[m i.a.w.t.TemporalAttemptExecution(lambda$getWorkerThread$2):158 - Completing future exceptionally...
io.airbyte.workers.WorkerException: Error while getting checking connection.
at io.airbyte.workers.DefaultCheckConnectionWorker.run(DefaultCheckConnectionWorker.java:84) ~[io.airbyte-airbyte-workers-0.35.55-alpha.jar:?]
at io.airbyte.workers.DefaultCheckConnectionWorker.run(DefaultCheckConnectionWorker.java:27) ~[io.airbyte-airbyte-workers-0.35.55-alpha.jar:?]
at io.airbyte.workers.temporal.TemporalAttemptExecution.lambda$getWorkerThread$2(TemporalAttemptExecution.java:155) ~[io.airbyte-airbyte-workers-0.35.55-alpha.jar:?]
at java.lang.Thread.run(Thread.java:833) [?:?]
Caused by: io.airbyte.workers.WorkerException: Operation: [create] for kind: [Pod] with name: [source-github-sync-ea7da5c8-55d6-4d4d-a435-656586c8f876-0-ncbss] in namespace: [airbyte-cluster] failed.
at io.airbyte.workers.process.KubeProcessFactory.create(KubeProcessFactory.java:150) ~[io.airbyte-airbyte-workers-0.35.55-alpha.jar:?]
at io.airbyte.workers.process.AirbyteIntegrationLauncher.check(AirbyteIntegrationLauncher.java:58) ~[io.airbyte-airbyte-workers-0.35.55-alpha.jar:?]
at io.airbyte.workers.DefaultCheckConnectionWorker.run(DefaultCheckConnectionWorker.java:53) ~[io.airbyte-airbyte-workers-0.35.55-alpha.jar:?]
... 3 more
Caused by: io.fabric8.kubernetes.client.KubernetesClientException: Operation: [create] for kind: [Pod] with name: [source-github-sync-ea7da5c8-55d6-4d4d-a435-656586c8f876-0-ncbss] in namespace: [airbyte-cluster] failed.
at io.fabric8.kubernetes.client.KubernetesClientException.launderThrowable(KubernetesClientException.java:64) ~[kubernetes-client-5.3.1.jar:?]
at io.fabric8.kubernetes.client.KubernetesClientException.launderThrowable(KubernetesClientException.java:72) ~[kubernetes-client-5.3.1.jar:?]
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.create(BaseOperation.java:373) ~[kubernetes-client-5.3.1.jar:?]
at io.fabric8.kubernetes.client.utils.CreateOrReplaceHelper.createOrReplace(CreateOrReplaceHelper.java:53) ~[kubernetes-client-5.3.1.jar:?]
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.createOrReplace(BaseOperation.java:411) ~[kubernetes-client-5.3.1.jar:?]
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.createOrReplace(BaseOperation.java:86) ~[kubernetes-client-5.3.1.jar:?]
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.createOrReplace(BaseOperation.java:394) ~[kubernetes-client-5.3.1.jar:?]
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.createOrReplace(BaseOperation.java:86) ~[kubernetes-client-5.3.1.jar:?]
at io.airbyte.workers.process.KubePodProcess.<init>(KubePodProcess.java:509) ~[io.airbyte-airbyte-workers-0.35.55-alpha.jar:?]
at io.airbyte.workers.process.KubeProcessFactory.create(KubeProcessFactory.java:146) ~[io.airbyte-airbyte-workers-0.35.55-alpha.jar:?]
at io.airbyte.workers.process.AirbyteIntegrationLauncher.check(AirbyteIntegrationLauncher.java:58) ~[io.airbyte-airbyte-workers-0.35.55-alpha.jar:?]
at io.airbyte.workers.DefaultCheckConnectionWorker.run(DefaultCheckConnectionWorker.java:53) ~[io.airbyte-airbyte-workers-0.35.55-alpha.jar:?]
... 3 more
Caused by: java.io.FileNotFoundException: /root/.kube/config (No such file or directory)
at java.io.FileInputStream.open0(Native Method) ~[?:?]
at java.io.FileInputStream.open(FileInputStream.java:216) ~[?:?]
at java.io.FileInputStream.<init>(FileInputStream.java:157) ~[?:?]
at com.fasterxml.jackson.dataformat.yaml.YAMLFactory.createParser(YAMLFactory.java:354) ~[jackson-dataformat-yaml-2.13.1.jar:2.13.1]
at com.fasterxml.jackson.dataformat.yaml.YAMLFactory.createParser(YAMLFactory.java:15) ~[jackson-dataformat-yaml-2.13.1.jar:2.13.1]
at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3494) ~[jackson-databind-2.13.1.jar:2.13.1]
at io.fabric8.kubernetes.client.internal.KubeConfigUtils.parseConfig(KubeConfigUtils.java:42) ~[kubernetes-client-5.3.1.jar:?]
at io.fabric8.kubernetes.client.utils.TokenRefreshInterceptor.intercept(TokenRefreshInterceptor.java:44) ~[kubernetes-client-5.3.1.jar:?]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147) ~[okhttp-3.12.12.jar:?]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:121) ~[okhttp-3.12.12.jar:?]
at io.fabric8.kubernetes.client.utils.ImpersonatorInterceptor.intercept(ImpersonatorInterceptor.java:68) ~[kubernetes-client-5.3.1.jar:?]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147) ~[okhttp-3.12.12.jar:?]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:121) ~[okhttp-3.12.12.jar:?]
at io.fabric8.kubernetes.client.utils.HttpClientUtils.lambda$createApplicableInterceptors$6(HttpClientUtils.java:284) ~[kubernetes-client-5.3.1.jar:?]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147) ~[okhttp-3.12.12.jar:?]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:121) ~[okhttp-3.12.12.jar:?]
at okhttp3.RealCall.getResponseWithInterceptorChain(RealCall.java:257) ~[okhttp-3.12.12.jar:?]
at okhttp3.RealCall.execute(RealCall.java:93) ~[okhttp-3.12.12.jar:?]
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:485) ~[kubernetes-client-5.3.1.jar:?]
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:448) ~[kubernetes-client-5.3.1.jar:?]
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleCreate(OperationSupport.java:263) ~[kubernetes-client-5.3.1.jar:?]
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.handleCreate(BaseOperation.java:870) ~[kubernetes-client-5.3.1.jar:?]
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.create(BaseOperation.java:365) ~[kubernetes-client-5.3.1.jar:?]
at io.fabric8.kubernetes.client.utils.CreateOrReplaceHelper.createOrReplace(CreateOrReplaceHelper.java:53) ~[kubernetes-client-5.3.1.jar:?]
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.createOrReplace(BaseOperation.java:411) ~[kubernetes-client-5.3.1.jar:?]
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.createOrReplace(BaseOperation.java:86) ~[kubernetes-client-5.3.1.jar:?]
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.createOrReplace(BaseOperation.java:394) ~[kubernetes-client-5.3.1.jar:?]
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.createOrReplace(BaseOperation.java:86) ~[kubernetes-client-5.3.1.jar:?]
at io.airbyte.workers.process.KubePodProcess.<init>(KubePodProcess.java:509) ~[io.airbyte-airbyte-workers-0.35.55-alpha.jar:?]
at io.airbyte.workers.process.KubeProcessFactory.create(KubeProcessFactory.java:146) ~[io.airbyte-airbyte-workers-0.35.55-alpha.jar:?]
at io.airbyte.workers.process.AirbyteIntegrationLauncher.check(AirbyteIntegrationLauncher.java:58) ~[io.airbyte-airbyte-workers-0.35.55-alpha.jar:?]
at io.airbyte.workers.DefaultCheckConnectionWorker.run(DefaultCheckConnectionWorker.java:53) ~[io.airbyte-airbyte-workers-0.35.55-alpha.jar:?]
... 3 more
2022-03-22 18:05:12 �[32mINFO�[m i.a.w.t.TemporalAttemptExecution(get):131 - Stopping cancellation check scheduling...
2022-03-22 18:05:12 �[33mWARN�[m i.t.i.a.POJOActivityTaskHandler(activityFailureToResult):307 - Activity failure. ActivityId=6cef53ed-b919-3960-a333-19b38e96dc07, activityType=Run, attempt=1
java.util.concurrent.ExecutionException: io.airbyte.workers.WorkerException: Error while getting checking connection.
at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?]
at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?]
at io.airbyte.workers.temporal.TemporalAttemptExecution.get(TemporalAttemptExecution.java:129) ~[io.airbyte-airbyte-workers-0.35.55-alpha.jar:?]
at io.airbyte.workers.temporal.check.connection.CheckConnectionActivityImpl.run(CheckConnectionActivityImpl.java:78) ~[io.airbyte-airbyte-workers-0.35.55-alpha.jar:?]
at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?]
at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
at io.temporal.internal.activity.POJOActivityTaskHandler$POJOActivityInboundCallsInterceptor.execute(POJOActivityTaskHandler.java:214) ~[temporal-sdk-1.8.1.jar:?]
at io.temporal.internal.activity.POJOActivityTaskHandler$POJOActivityImplementation.execute(POJOActivityTaskHandler.java:180) ~[temporal-sdk-1.8.1.jar:?]
at io.temporal.internal.activity.POJOActivityTaskHandler.handle(POJOActivityTaskHandler.java:120) ~[temporal-sdk-1.8.1.jar:?]
at io.temporal.internal.worker.ActivityWorker$TaskHandlerImpl.handle(ActivityWorker.java:204) ~[temporal-sdk-1.8.1.jar:?]
at io.temporal.internal.worker.ActivityWorker$TaskHandlerImpl.handle(ActivityWorker.java:164) ~[temporal-sdk-1.8.1.jar:?]
at io.temporal.internal.worker.PollTaskExecutor.lambda$process$0(PollTaskExecutor.java:93) ~[temporal-sdk-1.8.1.jar:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]
at java.lang.Thread.run(Thread.java:833) [?:?]
Caused by: io.airbyte.workers.WorkerException: Error while getting checking connection.
at io.airbyte.workers.DefaultCheckConnectionWorker.run(DefaultCheckConnectionWorker.java:84) ~[io.airbyte-airbyte-workers-0.35.55-alpha.jar:?]
at io.airbyte.workers.DefaultCheckConnectionWorker.run(DefaultCheckConnectionWorker.java:27) ~[io.airbyte-airbyte-workers-0.35.55-alpha.jar:?]
at io.airbyte.workers.temporal.TemporalAttemptExecution.lambda$getWorkerThread$2(TemporalAttemptExecution.java:155) ~[io.airbyte-airbyte-workers-0.35.55-alpha.jar:?]
... 1 more
Caused by: io.airbyte.workers.WorkerException: Operation: [create] for kind: [Pod] with name: [source-github-sync-ea7da5c8-55d6-4d4d-a435-656586c8f876-0-ncbss] in namespace: [airbyte-cluster] failed.
at io.airbyte.workers.process.KubeProcessFactory.create(KubeProcessFactory.java:150) ~[io.airbyte-airbyte-workers-0.35.55-alpha.jar:?]
at io.airbyte.workers.process.AirbyteIntegrationLauncher.check(AirbyteIntegrationLauncher.java:58) ~[io.airbyte-airbyte-workers-0.35.55-alpha.jar:?]
at io.airbyte.workers.DefaultCheckConnectionWorker.run(DefaultCheckConnectionWorker.java:53) ~[io.airbyte-airbyte-workers-0.35.55-alpha.jar:?]
at io.airbyte.workers.DefaultCheckConnectionWorker.run(DefaultCheckConnectionWorker.java:27) ~[io.airbyte-airbyte-workers-0.35.55-alpha.jar:?]
at io.airbyte.workers.temporal.TemporalAttemptExecution.lambda$getWorkerThread$2(TemporalAttemptExecution.java:155) ~[io.airbyte-airbyte-workers-0.35.55-alpha.jar:?]
... 1 more
Caused by: io.fabric8.kubernetes.client.KubernetesClientException: Operation: [create] for kind: [Pod] with name: [source-github-sync-ea7da5c8-55d6-4d4d-a435-656586c8f876-0-ncbss] in namespace: [airbyte-cluster] failed.
at io.fabric8.kubernetes.client.KubernetesClientException.launderThrowable(KubernetesClientException.java:64) ~[kubernetes-client-5.3.1.jar:?]
at io.fabric8.kubernetes.client.KubernetesClientException.launderThrowable(KubernetesClientException.java:72) ~[kubernetes-client-5.3.1.jar:?]
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.create(BaseOperation.java:373) ~[kubernetes-client-5.3.1.jar:?]
at io.fabric8.kubernetes.client.utils.CreateOrReplaceHelper.createOrReplace(CreateOrReplaceHelper.java:53) ~[kubernetes-client-5.3.1.jar:?]
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.createOrReplace(BaseOperation.java:411) ~[kubernetes-client-5.3.1.jar:?]
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.createOrReplace(BaseOperation.java:86) ~[kubernetes-client-5.3.1.jar:?]
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.createOrReplace(BaseOperation.java:394) ~[kubernetes-client-5.3.1.jar:?]
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.createOrReplace(BaseOperation.java:86) ~[kubernetes-client-5.3.1.jar:?]
at io.airbyte.workers.process.KubePodProcess.<init>(KubePodProcess.java:509) ~[io.airbyte-airbyte-workers-0.35.55-alpha.jar:?]
at io.airbyte.workers.process.KubeProcessFactory.create(KubeProcessFactory.java:146) ~[io.airbyte-airbyte-workers-0.35.55-alpha.jar:?]
at io.airbyte.workers.process.AirbyteIntegrationLauncher.check(AirbyteIntegrationLauncher.java:58) ~[io.airbyte-airbyte-workers-0.35.55-alpha.jar:?]
at io.airbyte.workers.DefaultCheckConnectionWorker.run(DefaultCheckConnectionWorker.java:53) ~[io.airbyte-airbyte-workers-0.35.55-alpha.jar:?]
at io.airbyte.workers.DefaultCheckConnectionWorker.run(DefaultCheckConnectionWorker.java:27) ~[io.airbyte-airbyte-workers-0.35.55-alpha.jar:?]
at io.airbyte.workers.temporal.TemporalAttemptExecution.lambda$getWorkerThread$2(TemporalAttemptExecution.java:155) ~[io.airbyte-airbyte-workers-0.35.55-alpha.jar:?]
... 1 more
Caused by: java.io.FileNotFoundException: /root/.kube/config (No such file or directory)
at java.io.FileInputStream.open0(Native Method) ~[?:?]
at java.io.FileInputStream.open(FileInputStream.java:216) ~[?:?]
at java.io.FileInputStream.<init>(FileInputStream.java:157) ~[?:?]
at com.fasterxml.jackson.dataformat.yaml.YAMLFactory.createParser(YAMLFactory.java:354) ~[jackson-dataformat-yaml-2.13.1.jar:2.13.1]
at com.fasterxml.jackson.dataformat.yaml.YAMLFactory.createParser(YAMLFactory.java:15) ~[jackson-dataformat-yaml-2.13.1.jar:2.13.1]
at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:3494) ~[jackson-databind-2.13.1.jar:2.13.1]
at io.fabric8.kubernetes.client.internal.KubeConfigUtils.parseConfig(KubeConfigUtils.java:42) ~[kubernetes-client-5.3.1.jar:?]
at io.fabric8.kubernetes.client.utils.TokenRefreshInterceptor.intercept(TokenRefreshInterceptor.java:44) ~[kubernetes-client-5.3.1.jar:?]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147) ~[okhttp-3.12.12.jar:?]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:121) ~[okhttp-3.12.12.jar:?]
at io.fabric8.kubernetes.client.utils.ImpersonatorInterceptor.intercept(ImpersonatorInterceptor.java:68) ~[kubernetes-client-5.3.1.jar:?]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147) ~[okhttp-3.12.12.jar:?]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:121) ~[okhttp-3.12.12.jar:?]
at io.fabric8.kubernetes.client.utils.HttpClientUtils.lambda$createApplicableInterceptors$6(HttpClientUtils.java:284) ~[kubernetes-client-5.3.1.jar:?]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147) ~[okhttp-3.12.12.jar:?]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:121) ~[okhttp-3.12.12.jar:?]
at okhttp3.RealCall.getResponseWithInterceptorChain(RealCall.java:257) ~[okhttp-3.12.12.jar:?]
at okhttp3.RealCall.execute(RealCall.java:93) ~[okhttp-3.12.12.jar:?]
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:485) ~[kubernetes-client-5.3.1.jar:?]
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:448) ~[kubernetes-client-5.3.1.jar:?]
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleCreate(OperationSupport.java:263) ~[kubernetes-client-5.3.1.jar:?]
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.handleCreate(BaseOperation.java:870) ~[kubernetes-client-5.3.1.jar:?]
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.create(BaseOperation.java:365) ~[kubernetes-client-5.3.1.jar:?]
at io.fabric8.kubernetes.client.utils.CreateOrReplaceHelper.createOrReplace(CreateOrReplaceHelper.java:53) ~[kubernetes-client-5.3.1.jar:?]
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.createOrReplace(BaseOperation.java:411) ~[kubernetes-client-5.3.1.jar:?]
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.createOrReplace(BaseOperation.java:86) ~[kubernetes-client-5.3.1.jar:?]
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.createOrReplace(BaseOperation.java:394) ~[kubernetes-client-5.3.1.jar:?]
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.createOrReplace(BaseOperation.java:86) ~[kubernetes-client-5.3.1.jar:?]
at io.airbyte.workers.process.KubePodProcess.<init>(KubePodProcess.java:509) ~[io.airbyte-airbyte-workers-0.35.55-alpha.jar:?]
at io.airbyte.workers.process.KubeProcessFactory.create(KubeProcessFactory.java:146) ~[io.airbyte-airbyte-workers-0.35.55-alpha.jar:?]
at io.airbyte.workers.process.AirbyteIntegrationLauncher.check(AirbyteIntegrationLauncher.java:58) ~[io.airbyte-airbyte-workers-0.35.55-alpha.jar:?]
at io.airbyte.workers.DefaultCheckConnectionWorker.run(DefaultCheckConnectionWorker.java:53) ~[io.airbyte-airbyte-workers-0.35.55-alpha.jar:?]
at io.airbyte.workers.DefaultCheckConnectionWorker.run(DefaultCheckConnectionWorker.java:27) ~[io.airbyte-airbyte-workers-0.35.55-alpha.jar:?]
at io.airbyte.workers.temporal.TemporalAttemptExecution.lambda$getWorkerThread$2(TemporalAttemptExecution.java:155) ~[io.airbyte-airbyte-workers-0.35.55-alpha.jar:?]
... 1 more