feat: improve decompression middleware

closes #598

Author: yordis

lib/tesla/middleware/compression.ex

@@ -19,6 +19,8 @@ defmodule Tesla.Middleware.Compression do
   - `:format` - request compression format, `"gzip"` (default) or `"deflate"`
   """
 
+  require Logger
+
   @behaviour Tesla.Middleware
 
   @impl Tesla.Middleware
@@ -49,6 +51,26 @@ defmodule Tesla.Middleware.Compression do
     end
   end
 
+  defmacrop brotli_loaded? do
+    if Code.ensure_loaded?(:brotli) do
+      true
+    else
+      quote do
+        Code.ensure_loaded?(:brotli)
+      end
+    end
+  end
+
+  defmacrop ezstd_loaded? do
+    if Code.ensure_loaded?(:ezstd) do
+      true
+    else
+      quote do
+        Code.ensure_loaded?(:ezstd)
+      end
+    end
+  end
+
   defp compress_body(body, "gzip"), do: :zlib.gzip(body)
   defp compress_body(body, "deflate"), do: :zlib.zip(body)
 
@@ -61,13 +83,84 @@ defmodule Tesla.Middleware.Compression do
   def decompress({:error, reason}), do: {:error, reason}
 
   def decompress(env) do
+    codecs = compression_algorithms(Tesla.get_header(env, "content-encoding"))
+    {decompressed_body, unknown_codecs} = decompress_body(codecs, env.body, [])
+
+    env
+    |> put_decompressed_body(decompressed_body)
+    |> put_or_delete_content_encoding(unknown_codecs)
+  end
+
+  defp put_or_delete_content_encoding(env, []) do
+    Tesla.delete_header(env, "content-encoding")
+  end
+
+  defp put_or_delete_content_encoding(env, unknown_codecs) do
+    Tesla.put_header(env, "content-encoding", Enum.join(unknown_codecs, ", "))
+  end
+
+  defp decompress_body([gzip | rest], body, acc) when gzip in ["gzip", "x-gzip"] do
+    decompress_body(rest, :zlib.gunzip(body), acc)
+  end
+
+  defp decompress_body(["br" | rest], body, acc) do
+    if brotli_loaded?() do
+      {:ok, decompressed} = :brotli.decode(body)
+      decompress_body(rest, decompressed, acc)
+    else
+      Logger.debug(":brotli library not loaded, skipping brotli decompression")
+      decompress_body(rest, body, ["br" | acc])
+    end
+  end
+
+  defp decompress_body(["zstd" | rest], body, acc) do
+    if ezstd_loaded?() do
+      decompress_body(rest, :ezstd.decompress(body), acc)
+    else
+      Logger.debug(":ezstd library not loaded, skipping zstd decompression")
+      decompress_body(rest, body, ["zstd" | acc])
+    end
+  end
+
+  defp decompress_body(["identity" | rest], body, acc) do
+    decompress_body(rest, body, acc)
+  end
+
+  defp decompress_body([codec | rest], body, acc) do
+    Logger.debug("algorithm #{inspect(codec)} is not supported")
+    decompress_body(rest, body, [codec | acc])
+  end
+
+  defp decompress_body([], body, acc) do
+    {body, acc}
+  end
+
+  defp compression_algorithms(nil) do
+    []
+  end
+
+  defp compression_algorithms(value) do
+    value
+    |> String.downcase()
+    |> String.split(",", trim: true)
+    |> Enum.map(&String.trim/1)
+    |> Enum.reverse()
+  end
+
+  defp put_decompressed_body(env, body) do
+    content_length =
+      body
+      |> byte_size()
+      |> to_string()
+
     env
-    |> Tesla.put_body(decompress_body(env.body, Tesla.get_header(env, "content-encoding")))
+    |> Tesla.put_body(body)
+    |> Tesla.put_header("content-length", content_length)
   end
 
   defp decompress_body(<<31, 139, 8, _::binary>> = body, "gzip"), do: :zlib.gunzip(body)
   defp decompress_body(body, "deflate"), do: :zlib.unzip(body)
-  defp decompress_body(body, _content_encoding), do: body
+  defp decompress_body(_body, _content_encoding), do: nil
 end
 
 defmodule Tesla.Middleware.CompressRequest do

mix.exs

@@ -20,7 +20,8 @@ defmodule Tesla.Mixfile do
         plt_add_apps: [:mix, :inets, :idna, :ssl_verify_fun, :ex_unit],
         plt_add_deps: :project
       ],
-      docs: docs()
+      docs: docs(),
+      preferred_cli_env: [coveralls: :test, "coveralls.html": :test]
     ]
   end
 

test/support/test_support.ex

@@ -0,0 +1,7 @@
+defmodule TestSupport do
+  def gzip_headers(env) do
+    env.headers
+    |> Enum.map_join("|", fn {key, value} -> "#{key}: #{value}" end)
+    |> :zlib.gzip()
+  end
+end

test/tesla/middleware/compression_test.exs

@@ -69,6 +69,7 @@ defmodule Tesla.Middleware.CompressionTest do
 
   test "decompress response body (gzip)" do
     assert {:ok, env} = CompressionResponseClient.get("/response-gzip")
+    assert env.headers == [{"content-type", "text/plain"}, {"content-length", "17"}]
     assert env.body == "decompressed gzip"
   end
 
@@ -80,6 +81,7 @@ defmodule Tesla.Middleware.CompressionTest do
   test "return unchanged response for unsupported content-encoding" do
     assert {:ok, env} = CompressionResponseClient.get("/response-identity")
     assert env.body == "unchanged"
+    assert env.headers == [{"content-type", "text/plain"}, {"content-encoding", "identity"}]
   end
 
   defmodule CompressRequestDecompressResponseClient do
@@ -114,7 +116,8 @@ defmodule Tesla.Middleware.CompressionTest do
       {status, headers, body} =
         case env.url do
           "/" ->
-            {200, [{"content-type", "text/plain"}, {"content-encoding", "gzip"}], env.headers}
+            {200, [{"content-type", "text/plain"}, {"content-encoding", "gzip"}],
+             TestSupport.gzip_headers(env)}
         end
 
       {:ok, %{env | status: status, headers: headers, body: body}}
@@ -123,7 +126,7 @@ defmodule Tesla.Middleware.CompressionTest do
 
   test "Compression headers" do
     assert {:ok, env} = CompressionHeadersClient.get("/")
-    assert env.body == [{"accept-encoding", "gzip, deflate"}]
+    assert env.body == "accept-encoding: gzip, deflate"
   end
 
   defmodule DecompressResponseHeadersClient do
@@ -135,7 +138,8 @@ defmodule Tesla.Middleware.CompressionTest do
       {status, headers, body} =
         case env.url do
           "/" ->
-            {200, [{"content-type", "text/plain"}, {"content-encoding", "gzip"}], env.headers}
+            {200, [{"content-type", "text/plain"}, {"content-encoding", "gzip"}],
+             TestSupport.gzip_headers(env)}
         end
 
       {:ok, %{env | status: status, headers: headers, body: body}}
@@ -144,7 +148,7 @@ defmodule Tesla.Middleware.CompressionTest do
 
   test "Decompress response headers" do
     assert {:ok, env} = DecompressResponseHeadersClient.get("/")
-    assert env.body == [{"accept-encoding", "gzip, deflate"}]
+    assert env.body == "accept-encoding: gzip, deflate"
   end
 
   defmodule CompressRequestHeadersClient do