Merge pull request #55 from lokalise/feature/PERF-1401_add_too_big_request_warning

PERF-1401 Add too big response warning

Author: bodrovis

docs/additional_info/changelog.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 3.6.1 (16-May-2025)
+
+* Add a warning for big projects suggesting to use [async file downloads](https://developers.lokalise.com/reference/download-files-async) endpoint
+
 ## 3.6.0 (08-May-2025)
 
 * Add support for [Current contributor](https://developers.lokalise.com/reference/retrieve-me-as-a-contributor) (me) endpoint

lib/elixir_lokalise_api/endpoints/files.ex

@@ -20,11 +20,24 @@ defmodule ElixirLokaliseApi.Files do
   Downloads a translation bundle from the project.
   """
   def download(project_id, data) do
-    make_request(:post,
-      data: data,
-      url_params: url_params(project_id) ++ [{:_postfix, "download"}],
-      type: :raw
-    )
+    case make_request(:post,
+           data: data,
+           url_params: url_params(project_id) ++ [{:_postfix, "download"}],
+           type: :raw
+         ) do
+      {:ok, result} ->
+        if Map.has_key?(result, :_request_too_big) do
+          IO.warn("""
+          Your project is too big for sync download.
+          Please use async one (ElixirLokaliseApi.Files.download_async/2) instead.
+          """)
+        end
+
+        {:ok, result}
+
+      other ->
+        other
+    end
   end
 
   @doc """

lib/elixir_lokalise_api/processor.ex

@@ -39,7 +39,7 @@ defmodule ElixirLokaliseApi.Processor do
 
     case json do
       raw_data when type == :raw and status < 400 ->
-        {:ok, raw_data}
+        {:ok, create_raw(raw_data, response.headers)}
 
       data when type == :foreign_model and status < 400 ->
         {:ok, create_struct(:foreign_model, module, data)}
@@ -60,6 +60,13 @@ defmodule ElixirLokaliseApi.Processor do
     end
   end
 
+  defp create_raw(raw_data, headers) do
+    case get_header(headers, "x-response-too-big") do
+      [] -> raw_data
+      _ -> Map.put(raw_data, :_request_too_big, true)
+    end
+  end
+
   defp create_struct(:foreign_model, module, raw_data) do
     foreign_model = module.foreign_model()
 
@@ -102,8 +109,8 @@ defmodule ElixirLokaliseApi.Processor do
         [] ->
           acc
 
-        [list | _] ->
-          header_value = list |> elem(1)
+        [header_pair | _] ->
+          header_value = header_pair |> elem(1)
 
           parsed_value =
             case Integer.parse(header_value) do

mix.exs

@@ -2,7 +2,7 @@ defmodule ElixirLokaliseApi.MixProject do
   use Mix.Project
 
   @source_url "https://github.com/lokalise/elixir-lokalise-api"
-  @version "3.6.0"
+  @version "3.6.1"
 
   def project do
     [

mix.lock

@@ -5,7 +5,7 @@
   "dialyxir": {:hex, :dialyxir, "1.4.5", "ca1571ac18e0f88d4ab245f0b60fa31ff1b12cbae2b11bd25d207f865e8ae78a", [:mix], [{:erlex, ">= 0.2.7", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "b0fb08bb8107c750db5c0b324fa2df5ceaa0f9307690ee3c1f6ba5b9eb5d35c3"},
   "earmark_parser": {:hex, :earmark_parser, "1.4.44", "f20830dd6b5c77afe2b063777ddbbff09f9759396500cdbe7523efd58d7a339c", [:mix], [], "hexpm", "4778ac752b4701a5599215f7030989c989ffdc4f6df457c5f36938cc2d2a2750"},
   "erlex": {:hex, :erlex, "0.2.7", "810e8725f96ab74d17aac676e748627a07bc87eb950d2b83acd29dc047a30595", [:mix], [], "hexpm", "3ed95f79d1a844c3f6bf0cea61e0d5612a42ce56da9c03f01df538685365efb0"},
-  "ex_doc": {:hex, :ex_doc, "0.37.3", "f7816881a443cd77872b7d6118e8a55f547f49903aef8747dbcb345a75b462f9", [:mix], [{:earmark_parser, "~> 1.4.42", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "e6aebca7156e7c29b5da4daa17f6361205b2ae5f26e5c7d8ca0d3f7e18972233"},
+  "ex_doc": {:hex, :ex_doc, "0.38.1", "bae0a0bd5b5925b1caef4987e3470902d072d03347114ffe03a55dbe206dd4c2", [:mix], [{:earmark_parser, "~> 1.4.44", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "754636236d191b895e1e4de2ebb504c057fe1995fdfdd92e9d75c4b05633008b"},
   "excoveralls": {:hex, :excoveralls, "0.18.5", "e229d0a65982613332ec30f07940038fe451a2e5b29bce2a5022165f0c9b157e", [:mix], [{:castore, "~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "523fe8a15603f86d64852aab2abe8ddbd78e68579c8525ae765facc5eae01562"},
   "exjsx": {:hex, :exjsx, "4.0.0", "60548841e0212df401e38e63c0078ec57b33e7ea49b032c796ccad8cde794b5c", [:mix], [{:jsx, "~> 2.8.0", [hex: :jsx, repo: "hexpm", optional: false]}], "hexpm", "32e95820a97cffea67830e91514a2ad53b888850442d6d395f53a1ac60c82e07"},
   "exvcr": {:hex, :exvcr, "0.17.1", "3bae83d698a464a48212ad87c8ea4bcfb6bd76d53b937129472764e557616228", [:mix], [{:exjsx, "~> 4.0", [hex: :exjsx, repo: "hexpm", optional: false]}, {:finch, "~> 0.16", [hex: :finch, repo: "hexpm", optional: true]}, {:httpoison, "~> 1.0 or ~> 2.0", [hex: :httpoison, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.4", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:meck, "~> 1.0", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm", "a39d86980da183011366a878972e0ed43a5441814b701edd2e11360564e7bcab"},

test/elixir_lokalise_api/endpoints/files_test.exs

@@ -1,3 +1,5 @@
+import ExUnit.CaptureIO
+
 defmodule ElixirLokaliseApi.FilesTest do
   use ExUnit.Case, async: true
   use ExVCR.Mock, adapter: ExVCR.Adapter.Hackney
@@ -82,6 +84,42 @@ defmodule ElixirLokaliseApi.FilesTest do
     end
   end
 
+  test "downloads files with too big response" do
+    use_cassette "files_download_response_too_big" do
+      data = %{
+        format: "json",
+        original_filenames: true
+      }
+
+      warning =
+        capture_io(:stderr, fn ->
+          {:ok, %{} = resp} = Files.download(@project_id, data)
+          assert String.contains?(resp.bundle_url, "Demo_Phoenix")
+          assert resp.project_id == @project_id
+          assert resp._request_too_big
+        end)
+
+      assert warning =~ "Your project is too big for sync download"
+    end
+  end
+
+  test "downloads file (error)" do
+    use_cassette "files_download_error" do
+      data = %{
+        format: "json",
+        original_filenames: true
+      }
+
+      {:error, resp} = Files.download(@project_id, data)
+
+      {msg_data, code} = resp
+      message = msg_data.error.message
+
+      assert code == 500
+      assert message == "Fail"
+    end
+  end
+
   test "downloads files asynchronously" do
     process_id = "1efed526-cd7c-6f2e-9db9-360c3c31288f"
     project_id = "6504960967ab53d45e0ed7.15877499"

test/fixtures/cassettes/files_download_error.json

@@ -0,0 +1,39 @@
+[
+  {
+    "request": {
+      "body": "{\"format\":\"json\",\"original_filenames\":true}",
+      "headers": {
+        "X-Api-Token": "***",
+        "Accept": "application/json",
+        "User-Agent": "elixir-lokalise-api package/3.5.0"
+      },
+      "method": "post",
+      "options": [],
+      "request_body": "",
+      "url": "https://api.lokalise.com/api2/projects/803826145ba90b42d5d860.46800099/files/download"
+    },
+    "response": {
+      "binary": false,
+      "body": "{\"error\":{\"message\":\"Fail\",\"code\":500}}",
+      "headers": {
+        "Server": "nginx",
+        "Content-Type": "application/json",
+        "Transfer-Encoding": "chunked",
+        "Connection": "keep-alive",
+        "Vary": "Accept-Encoding",
+        "Cache-Control": "max-age=0, must-revalidate, no-cache, no-store, private",
+        "Date": "Tue, 09 Mar 2021 15:30:44 GMT",
+        "Pragma": "no-cache",
+        "X-Content-Type-Options": "nosniff",
+        "X-Frame-Options": "deny",
+        "X-XSS-Protection": "1; mode=block",
+        "Strict-Transport-Security": "max-age=31536000",
+        "Referrer-Policy": "origin",
+        "Expires": "Tue, 09 Mar 2021 15:30:44 GMT",
+        "Access-Control-Allow-Origin": "*",
+        "Access-Control-Allow-Headers": "Content-Type"
+      },
+      "status_code": 500
+    }
+  }
+]

test/fixtures/cassettes/files_download_response_too_big.json

@@ -0,0 +1,41 @@
+[
+  {
+    "request": {
+      "body": "{\"format\":\"json\",\"original_filenames\":true}",
+      "headers": {
+        "X-Api-Token": "***",
+        "Accept": "application/json",
+        "User-Agent": "elixir-lokalise-api package/3.5.0"
+      },
+      "method": "post",
+      "options": [],
+      "request_body": "",
+      "url": "https://api.lokalise.com/api2/projects/803826145ba90b42d5d860.46800099/files/download"
+    },
+    "response": {
+      "binary": false,
+      "body": "{\"project_id\":\"803826145ba90b42d5d860.46800099\",\"branch\":\"master\",\"bundle_url\":\"https:\\/\\/s3-eu-west-1.amazonaws.com\\/lokalise-assets\\/files\\/export\\/803826145ba90b42d5d860.46800099\\/dd272dd041482e636bdc30142a8944b6\\/Demo_Phoenix-locale.zip\"}",
+      "headers": {
+        "Server": "nginx",
+        "Content-Type": "application/json",
+        "Transfer-Encoding": "chunked",
+        "Connection": "keep-alive",
+        "Vary": "Accept-Encoding",
+        "Cache-Control": "max-age=0, must-revalidate, no-cache, no-store, private",
+        "Date": "Tue, 09 Mar 2021 15:30:44 GMT",
+        "Pragma": "no-cache",
+        "X-Content-Type-Options": "nosniff",
+        "X-Frame-Options": "deny",
+        "X-XSS-Protection": "1; mode=block",
+        "X-Response-Too-Big": "true",
+        "Strict-Transport-Security": "max-age=31536000",
+        "Referrer-Policy": "origin",
+        "Expires": "Tue, 09 Mar 2021 15:30:44 GMT",
+        "Access-Control-Allow-Origin": "*",
+        "Access-Control-Allow-Headers": "Content-Type"
+      },
+      "status_code": 200,
+      "type": "ok"
+    }
+  }
+]