:hackney.stream_body consistently times out in :hackney > 1.18.1

[halfdan]

Using the following code:

defmodule Upload do
  @multipart_minimum 5 * 1024 ** 2
  @recv_timeout to_timeout(second: 5)
  
  def stream(url, retry \\ 0)
  def stream({:ok, client}, _), do: Stream.resource(fn -> client end, &continue_stream/1, fn _ -> nil end)
  def stream({:error, url}, retry) when retry < 3, do: stream(url, retry + 1)
  def stream({:error, _}, _), do: []

  def stream(%URI{} = uri, retry) do
    uri
    |> URI.to_string()
    |> stream(retry)
  end

  def stream(url, retry) when is_binary(url) do
    case :hackney.get(url, [], "", recv_timeout: @recv_timeout) do
      {:ok, _, _, client} ->
        stream({:ok, client}, retry)

      _ ->
        stream({:error, url}, retry)
    end
  end

  defp continue_stream({:halt, client}), do: {:halt, client}

  defp continue_stream(client) do
    with {:ok, data} <- get_chunk(client), do: ensure_size(data, client)
  end

  defp ensure_size(data, client) when byte_size(data) >= @multipart_minimum, do: {[data], client}

  defp ensure_size(data, client) do
    case get_chunk(client) do
      {:ok, chunk} ->
        ensure_size(data <> chunk, client)

      _ ->
        {[data], {:halt, client}}
    end
  end

  defp get_chunk(client) do
    case :hackney.stream_body(client) do
      {:ok, data} ->
        {:ok, data}

      :done ->
        {:halt, client}

      {:error, error} ->
        raise ArgumentError, message: to_string(error)
    end
  end

end

and calling it with:

url = "https://esahubble.org/media/archives/images/publicationtiff40k/heic1502a.tif"
Download.stream(url)
|> dbg()
|> Enum.count()

Runs straight into the recv_timeout:

** (ArgumentError) timeout
    #cell:wuuajfh4jvvhuhlm:54: Download.get_chunk/1
    #cell:wuuajfh4jvvhuhlm:35: Download.ensure_size/2
    (elixir 1.18.2) lib/stream.ex:1571: Stream.do_resource/5
    (elixir 1.18.2) lib/enum.ex:707: Enum.count/1
    #cell:haftwaxp2sobsqt7:4: (file)

1.18.1 handled this just fine and get_chunk would be called many times whereas starting with 1.18.2 get_chunk gets called once and then hangs until it hits the timeout.

[benoitc]

Thanks for the report, I am looking at it. when you say > 1.18.1 did you also tried with recent version > 1.21 ?

[halfdan]

I tried all new versions including 1.23.0. It's this commit: 5e74354. If I pin it to the commit right before the above code works fine.

There seems to be a related bug that was also introduced in #710 with a potential fix in 1a13c8f, but that doesn't address the issue described here.

[benoitc]

should be fixed in master

[rNoz]

Tested it yesterday and it is still failing.

url = "https://esahubble.org/media/archives/images/publicationtiff40k/heic1502a.tif"
# It is downloading it, but since OP module is Upload:
Upload.stream(url) |> dbg() |> Enum.count()