another attempt to fix copyFrom

Signed-off-by: Jeromy Cannon <[email protected]>

Author: jeromy-cannon

src/core/k8.mjs

@@ -567,21 +567,22 @@ export class K8 {
       const destPath = path.join(destDir, srcFile)
 
       // download the tar file to a temp location
-      const tmpFile = this._tempFileFor(srcFile)
+      const tmpFile = `${this._tempFileFor(srcFile)}.tar`
 
       const self = this
       return new Promise((resolve, reject) => {
         const execInstance = new k8s.Exec(this.kubeConfig)
         const command = ['tar', 'cf', '-', '-C', srcDir, srcFile]
         const outputFileStream = fs.createWriteStream(tmpFile)
-        const outputPassthroughStream = new stream.PassThrough({ highWaterMark: 1024 * 1024 })
+        const outputPassthroughStream = new stream.PassThrough({ highWaterMark: 10 * 1024 * 1024 })
         const errStream = new stream.PassThrough()
         let additionalErrorMessageDetail = ''
 
         // Use pipe() to automatically handle backpressure between streams
         outputPassthroughStream.pipe(outputFileStream)
 
         outputPassthroughStream.on('data', (chunk) => {
+          this.logger.debug(`received chunk size=${chunk.length}`)
           const canWrite = outputFileStream.write(chunk) // Write chunk to file and check if buffer is full
 
           if (!canWrite) {
@@ -605,42 +606,44 @@ export class K8 {
           null,
           false,
           async ({ status }) => {
-            this.logger.debug(`copyFrom.callback(status)=${status}`)
-            outputFileStream.end()
-            outputFileStream.close(() => {
-              this.logger.debug(`finished closing writerStream copying from ${podName}:${srcDir}/${srcFile} to ${destPath}`)
-            })
             if (status === 'Failure') {
               self._deleteTempFile(tmpFile)
+              return reject(new FullstackTestingError('Tar command failed with status Failure'))
             }
+            this.logger.debug(`copyFrom.callback(status)=${status}`)
           })
           .then(conn => {
             conn.on('close', async (code, reason) => {
               if (code !== 1000) { // code 1000 is the success code
                 return reject(new FullstackTestingError(`failed copying from ${podName}:${srcDir}/${srcFile} to ${destPath} because of error (${code}): ${reason}`))
               }
 
-              try {
-                // extract the downloaded file
-                await tar.x({
-                  file: tmpFile,
-                  cwd: destDir
-                })
-                self._deleteTempFile(tmpFile)
-
-                const stat = fs.statSync(destPath)
-                if (stat && stat.size === srcFileSize) {
-                  return resolve(true)
-                } else {
-                  if (!stat) {
-                    additionalErrorMessageDetail = ', statSync returned no file status for the destination file'
+              outputFileStream.end()
+              outputFileStream.close(async () => {
+                this.logger.debug(`finished closing writerStream copying from ${podName}:${srcDir}/${srcFile} to ${destPath}`)
+
+                try {
+                  // extract the downloaded file
+                  await tar.x({
+                    file: tmpFile,
+                    cwd: destDir
+                  })
+                  self._deleteTempFile(tmpFile)
+
+                  const stat = fs.statSync(destPath)
+                  if (stat && stat.size === srcFileSize) {
+                    return resolve(true)
                   } else {
-                    additionalErrorMessageDetail = `, stat.size=${stat.size} != srcFileSize=${srcFileSize}`
+                    if (!stat) {
+                      additionalErrorMessageDetail = ', statSync returned no file status for the destination file'
+                    } else {
+                      additionalErrorMessageDetail = `, stat.size=${stat.size} != srcFileSize=${srcFileSize}`
+                    }
                   }
+                } catch (e) {
+                  return reject(new FullstackTestingError(`failed copying from ${podName}:${srcDir}/${srcFile} to ${destPath} to extract file: ${destPath}`, e))
                 }
-              } catch (e) {
-                return reject(new FullstackTestingError(`failed copying from ${podName}:${srcDir}/${srcFile} to ${destPath} to extract file: ${destPath}`, e))
-              }
+              })
 
               return reject(new FullstackTestingError(`failed copying from ${podName}:${srcDir}/${srcFile} to ${destPath} to download file completely: ${destPath}${additionalErrorMessageDetail}`))
             })
@@ -653,6 +656,7 @@ export class K8 {
           })
 
         errStream.on('data', (data) => {
+          this.logger.error(`error encountered copying from ${podName}:${srcDir}/${srcFile} to ${destPath}, error: ${data.toString()}`)
           return reject(new FullstackTestingError(`error encountered copying from ${podName}:${srcDir}/${srcFile} to ${destPath}, error: ${data.toString()}`))
         })
 
@@ -673,12 +677,10 @@ export class K8 {
         })
 
         outputFileStream.on('finish', () => {
-          outputFileStream.end()
           this.logger.debug(`stopping copy, writerStream has finished for copying from ${podName}:${srcDir}/${srcFile} to ${destPath}`)
         })
 
         outputPassthroughStream.on('finish', () => {
-          outputFileStream.end()
           this.logger.debug(`stopping copy, writerPassthroughStream has finished for copying from ${podName}:${srcDir}/${srcFile} to ${destPath}`)
         })
       })