Skip to content

Commit b1f48e9

Browse files
ruslandogaruslandoga
authored
Add Typesense (#44)
1 parent d6cadef commit b1f48e9

File tree

14 files changed

+557
-4
lines changed

14 files changed

+557
-4
lines changed

‎.github/workflows/main.yml

+4-1
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,9 @@ jobs:
2424
- name: Check mix format
2525
run: mix format --check-formatted
2626

27+
- name: Start Typesense
28+
run: docker compose up -d typesense
29+
2730
- name: Run tests
2831
run: |
29-
mix test
32+
mix test --include typesense

‎compose.yml

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
services:
2+
typesense:
3+
image: typesense/typesense:27.1
4+
command: --data-dir /tmp --api-key=hexdocs
5+
ports:
6+
- 8108:8108

‎config/config.exs

+4
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,13 @@ config :hexdocs,
44
port: "4002",
55
hexpm_url: "http://localhost:4000",
66
hexpm_secret: "2cd6d09334d4b00a2be4d532342b799b",
7+
typesense_url: "http://localhost:8108",
8+
typesense_api_key: "hexdocs",
9+
typesense_collection: "hexdocs",
710
hexpm_impl: Hexdocs.Hexpm.Impl,
811
store_impl: Hexdocs.Store.Local,
912
cdn_impl: Hexdocs.CDN.Local,
13+
search_impl: Hexdocs.Search.Local,
1014
source_repo_impl: Hexdocs.SourceRepo.GitHub,
1115
tmp_dir: "tmp",
1216
queue_id: "test",

‎config/dev.exs

+2-1
Original file line numberDiff line numberDiff line change
@@ -5,4 +5,5 @@ config :hexdocs,
55
hexpm_url: "http://localhost:4000",
66
hexpm_impl: Hexdocs.Hexpm.Impl,
77
store_impl: Hexdocs.Store.Local,
8-
cdn_impl: Hexdocs.CDN.Local
8+
cdn_impl: Hexdocs.CDN.Local,
9+
search_impl: Hexdocs.Search.Local

‎config/prod.exs

+1
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ config :hexdocs,
44
hexpm_impl: Hexdocs.Hexpm.Impl,
55
store_impl: Hexdocs.Store.Impl,
66
cdn_impl: Hexdocs.CDN.Fastly,
7+
search_impl: Hexdocs.Search.Typesense,
78
queue_producer: BroadwaySQS.Producer,
89
gcs_put_debounce: 3000
910

‎config/runtime.exs

+3
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,9 @@ if config_env() == :prod do
55
port: System.fetch_env!("HEXDOCS_PORT"),
66
hexpm_url: System.fetch_env!("HEXDOCS_HEXPM_URL"),
77
hexpm_secret: System.fetch_env!("HEXDOCS_HEXPM_SECRET"),
8+
typesense_url: System.fetch_env!("HEXDOCS_TYPESENSE_URL"),
9+
typesense_api_key: System.fetch_env!("HEXDOCS_TYPESENSE_API_KEY"),
10+
typesense_collection: System.fetch_env!("HEXDOCS_TYPESENSE_COLLECTION"),
811
fastly_key: System.fetch_env!("HEXDOCS_FASTLY_KEY"),
912
fastly_hexdocs: System.fetch_env!("HEXDOCS_FASTLY_HEXDOCS"),
1013
queue_id: System.fetch_env!("HEXDOCS_QUEUE_ID"),

‎config/test.exs

+1
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ config :hexdocs,
66
hexpm_impl: Hexdocs.HexpmMock,
77
store_impl: Hexdocs.Store.Local,
88
cdn_impl: Hexdocs.CDN.Local,
9+
search_impl: Hexdocs.Search.Local,
910
source_repo_impl: Hexdocs.SourceRepo.Mock
1011

1112
config :logger, level: :warning

‎lib/hexdocs/http.ex

+4
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,10 @@ defmodule Hexdocs.HTTP do
2323
|> read_response()
2424
end
2525

26+
def post(url, headers, body, opts \\ []) do
27+
:hackney.post(url, headers, body, opts)
28+
end
29+
2630
def delete(url, headers) do
2731
:hackney.delete(url, headers)
2832
|> read_response()

‎lib/hexdocs/queue.ex

+17-1
Original file line numberDiff line numberDiff line change
@@ -125,8 +125,12 @@ defmodule Hexdocs.Queue do
125125
update_package_sitemap(repository, key, package, files)
126126
end
127127

128+
if repository == "hexpm" do
129+
update_search_index(key, package, version, files)
130+
end
131+
128132
elapsed = System.os_time(:millisecond) - start
129-
Logger.info("FINISHED UPLOADING DOCS #{key} #{elapsed}ms")
133+
Logger.info("FINISHED UPLOADING AND INDEXING DOCS #{key} #{elapsed}ms")
130134

131135
{:error, reason} ->
132136
Logger.error("Failed unpack #{repository}/#{package} #{version}: #{reason}")
@@ -149,6 +153,10 @@ defmodule Hexdocs.Queue do
149153
Hexdocs.Bucket.delete(repository, package, version, all_versions)
150154
update_index_sitemap(repository, key)
151155

156+
if repository == "hexpm" do
157+
Hexdocs.Search.delete(package, version)
158+
end
159+
152160
elapsed = System.os_time(:millisecond) - start
153161
Logger.info("FINISHED DELETING DOCS #{key} #{elapsed}ms")
154162
:ok
@@ -228,6 +236,14 @@ defmodule Hexdocs.Queue do
228236
:ok
229237
end
230238

239+
defp update_search_index(key, package, version, files) do
240+
with {proglang, items} <- Hexdocs.Search.find_search_items(package, version, files) do
241+
Logger.info("UPDATING SEARCH INDEX #{key}")
242+
Hexdocs.Search.index(package, version, proglang, items)
243+
Logger.info("UPDATED SEARCH INDEX #{key}")
244+
end
245+
end
246+
231247
@doc false
232248
def paths_for_sitemaps() do
233249
key_regex = ~r"docs/(.*)-(.*).tar.gz$"

‎lib/hexdocs/search/local.ex

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
defmodule Hexdocs.Search.Local do
2+
@behaviour Hexdocs.Search
3+
4+
@impl true
5+
def index(_package, _version, _proglang, _items), do: :ok
6+
7+
@impl true
8+
def delete(_package, _version), do: :ok
9+
end

‎lib/hexdocs/search/search.ex

+93
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
defmodule Hexdocs.Search do
2+
require Logger
3+
4+
@type package :: String.t()
5+
@type version :: Version.t()
6+
@type proglang :: String.t()
7+
@type search_items :: [map]
8+
9+
@callback index(package, version, proglang, search_items) :: :ok
10+
@callback delete(package, version) :: :ok
11+
12+
defp impl, do: Application.fetch_env!(:hexdocs, :search_impl)
13+
14+
@spec index(package, version, proglang, search_items) :: :ok
15+
def index(package, version, proglang, search_items) do
16+
impl().index(package, version, proglang, search_items)
17+
end
18+
19+
@spec delete(package, version) :: :ok
20+
def delete(package, version) do
21+
impl().delete(package, version)
22+
end
23+
24+
@spec find_search_items(package, version, [{Path.t(), content :: iodata}]) ::
25+
{proglang, search_items} | nil
26+
def find_search_items(package, version, files) do
27+
search_data_js =
28+
Enum.find_value(files, fn {path, content} ->
29+
case Path.basename(path) do
30+
"search_data-" <> _digest -> content
31+
_other -> nil
32+
end
33+
end)
34+
35+
unless search_data_js do
36+
Logger.info("Failed to find search data for #{package} #{version}")
37+
end
38+
39+
search_data_json =
40+
case search_data_js do
41+
"searchData=" <> json ->
42+
json
43+
44+
_ when is_binary(search_data_js) ->
45+
Logger.error("Unexpected search_data format for #{package} #{version}")
46+
nil
47+
48+
nil ->
49+
nil
50+
end
51+
52+
search_data =
53+
if search_data_json do
54+
try do
55+
:json.decode(search_data_json)
56+
catch
57+
_kind, reason ->
58+
Logger.error(
59+
"Failed to decode search data json for #{package} #{version}: " <>
60+
inspect(reason)
61+
)
62+
63+
nil
64+
end
65+
end
66+
67+
case search_data do
68+
%{"items" => [_ | _] = search_items} ->
69+
proglang = Map.get(search_data, "proglang") || proglang(search_items)
70+
{proglang, search_items}
71+
72+
nil ->
73+
nil
74+
75+
_ ->
76+
Logger.error(
77+
"Failed to extract search items and proglang from search data for #{package} #{version}"
78+
)
79+
80+
nil
81+
end
82+
end
83+
84+
defp proglang(search_items) do
85+
if Enum.any?(search_items, &elixir_module?/1), do: "elixir", else: "erlang"
86+
end
87+
88+
defp elixir_module?(%{"type" => "module", "title" => <<first_letter, _::binary>>})
89+
when first_letter in ?A..?Z,
90+
do: true
91+
92+
defp elixir_module?(_), do: false
93+
end

‎lib/hexdocs/search/typesense.ex

+105
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
defmodule Hexdocs.Search.Typesense do
2+
@moduledoc false
3+
require Logger
4+
alias Hexdocs.HTTP
5+
6+
@behaviour Hexdocs.Search
7+
8+
@impl true
9+
def index(package, version, proglang, search_items) do
10+
full_package = full_package(package, version)
11+
12+
ndjson =
13+
Enum.map(search_items, fn item ->
14+
json =
15+
Map.take(item, ["type", "ref", "title", "doc"])
16+
|> Map.put("package", full_package)
17+
|> Map.put("proglang", proglang)
18+
|> :json.encode()
19+
20+
[json, ?\n]
21+
end)
22+
23+
url = url("collections/#{collection()}/documents/import?action=create")
24+
headers = [{"x-typesense-api-key", api_key()}]
25+
26+
case HTTP.post(url, headers, ndjson, [:with_body]) do
27+
{:ok, 200, _resp_headers, ndjson} ->
28+
ndjson
29+
|> String.split("\n")
30+
|> Enum.each(fn json ->
31+
case :json.decode(json) do
32+
%{"success" => true} ->
33+
:ok
34+
35+
%{"success" => false, "error" => error, "document" => document} ->
36+
Logger.error(
37+
"Failed to index search item for #{package} #{version} for document #{inspect(document)}: #{inspect(error)}"
38+
)
39+
end
40+
end)
41+
42+
{:ok, status, _resp_headers, _body} ->
43+
Logger.error("Failed to index search items for #{package} #{version}: status=#{status}")
44+
45+
{:error, reason} ->
46+
Logger.error("Failed to index search items #{package} #{version}: #{inspect(reason)}")
47+
end
48+
end
49+
50+
@impl true
51+
def delete(package, version) do
52+
full_package = full_package(package, version)
53+
54+
query = URI.encode_query([{"filter_by", "package:#{full_package}"}])
55+
url = url("collections/#{collection()}/documents?" <> query)
56+
headers = [{"x-typesense-api-key", api_key()}]
57+
58+
case HTTP.delete(url, headers) do
59+
{:ok, 200, _resp_headers, _body} ->
60+
:ok
61+
62+
{:ok, status, _resp_headers, _body} ->
63+
Logger.error("Failed to delete search items for #{package} #{version}: status=#{status}")
64+
65+
{:error, reason} ->
66+
Logger.error(
67+
"Failed to delete search items for #{package} #{version}: #{inspect(reason)}"
68+
)
69+
end
70+
end
71+
72+
@spec collection :: String.t()
73+
def collection do
74+
Application.fetch_env!(:hexdocs, :typesense_collection)
75+
end
76+
77+
@spec collection_schema :: map
78+
def collection_schema(collection \\ collection()) do
79+
%{
80+
"fields" => [
81+
%{"facet" => true, "name" => "proglang", "type" => "string"},
82+
%{"facet" => true, "name" => "type", "type" => "string"},
83+
%{"name" => "title", "type" => "string"},
84+
%{"name" => "doc", "type" => "string"},
85+
%{"facet" => true, "name" => "package", "type" => "string"}
86+
],
87+
"name" => collection,
88+
"token_separators" => [".", "_", "-", " ", ":", "@", "/"]
89+
}
90+
end
91+
92+
@spec api_key :: String.t()
93+
def api_key do
94+
Application.fetch_env!(:hexdocs, :typesense_api_key)
95+
end
96+
97+
defp full_package(package, version) do
98+
"#{package}-#{version}"
99+
end
100+
101+
defp url(path) do
102+
base_url = Application.fetch_env!(:hexdocs, :typesense_url)
103+
Path.join(base_url, path)
104+
end
105+
end

0 commit comments

Comments
 (0)