WIP gh-71 : implemented the option for stream, and some minimal test

Author: Shakadak

CHANGELOG.md

@@ -6,6 +6,10 @@
 
 * Improvement on the doc
 
+### ADDED
+
+* Added option to raise on DTD definitions
+
 ## [0.6.6] (2019-02-24)
 
 * small bugfix: Fix compilation warnings on newer versions of Elixir

lib/sweet_xml.ex

@@ -227,7 +227,8 @@ defmodule SweetXml do
   """
   def parse(doc, opts \\ []) do
     ets = :ets.new(nil, [])
-    {dtd_arg, opts} = Keyword.pop(opts, :dtd, :all)
+    dtd_arg = :proplists.get_value(:dtd, opts, :all)
+    opts = :proplists.delete(:dtd, opts)
     opts = SweetXml.Options.handle_dtd(dtd_arg).(ets) ++ opts
     try do
       do_parse(doc, opts)
@@ -298,10 +299,9 @@ defmodule SweetXml do
   def stream_tags(doc, tags, options \\ []) do
     tags = if is_atom(tags), do: [tags], else: tags
 
-    {discard_tags, xmerl_options} = if options[:discard] do
-      {options[:discard], Keyword.delete(options, :discard)}
-    else
-      {[], options}
+    {discard_tags, xmerl_options} = case :proplists.lookup(:discard, options) do
+      {:discard, tags} -> {tags, :proplists.delete(:discard, options)}
+      :none -> {[], options}
     end
 
     doc |> stream(fn emit ->
@@ -369,22 +369,32 @@ defmodule SweetXml do
     Stream.resource fn ->
       {parent, ref} = waiter = {self(), make_ref()}
       opts = options_callback.(fn e -> send(parent, {:event, ref, e}) end)
+
+      ets = :ets.new(nil, [:public])
+      dtd_arg = :proplists.get_value(:dtd, opts, :all)
+      opts = :proplists.delete(:dtd, opts)
+      opts = SweetXml.Options.handle_dtd(dtd_arg).(ets) ++ opts
+
       pid = spawn_link fn -> :xmerl_scan.string('', opts ++ continuation_opts(doc, waiter)) end
-      {ref, pid, Process.monitor(pid)}
-    end, fn {ref, pid, monref} = acc ->
+      {ref, pid, Process.monitor(pid), ets}
+    end, fn {ref, pid, monref, ets} = acc ->
       receive do
         {:DOWN, ^monref, _, _, _} ->
-          {:halt, :parse_ended} ## !!! maybe do something when reason !== :normal
+          {:halt, {:parse_ended, ets}} ## !!! maybe do something when reason !== :normal
         {:event, ^ref, event} ->
           {[event], acc}
         {:wait, ^ref} ->
           send(pid, {:continue, ref})
           {[], acc}
       end
     end, fn
-      :parse_ended -> :ok
-      {ref, pid, monref} ->
+      {:parse_ended, ets} ->
+        _ = :ets.delete(ets)
+        :ok
+
+      {ref, pid, monref, ets} ->
         Process.demonitor(monref)
+        _ = :ets.delete(ets)
         flush_halt(pid, ref)
     end
   end

mix.exs

@@ -17,7 +17,7 @@ defmodule SweetXml.Mixfile do
 
   def application do
     [
-      applications: [:xmerl]
+      applications: [:logger, :xmerl]
     ]
   end
 

test/issue_71_test.exs

@@ -1,7 +1,7 @@
 defmodule Issue71Test do
   use ExUnit.Case
 
-  test "read /etc/passwd with dtd: :none" do
+  test "raise on reading /etc/passwd with dtd: :none" do
     sneaky_xml = """
     <?xml version=\"1.0\" encoding=\"UTF-8\"?>
     <!DOCTYPE foo [ <!ELEMENT foo ANY >
@@ -10,7 +10,7 @@ defmodule Issue71Test do
     """
 
     assert {:fatal, {{:error_fetching_DTD, {_, _}}, _file, _line, _col}} =
-      catch_exit(SweetXml.parse(sneaky_xml, dtd: :none))
+      catch_exit(SweetXml.parse(sneaky_xml, dtd: :none, quiet: true))
   end
 
   test "raise on billion_laugh.xml with dtd: :none" do
@@ -19,4 +19,25 @@ defmodule Issue71Test do
       SweetXml.parse(dangerous_xml, dtd: :none)
     end
   end
+
+  test "stream: raise on reading /etc/passwd with dtd: :none" do
+    sneaky_xml = """
+    <?xml version=\"1.0\" encoding=\"UTF-8\"?>
+    <!DOCTYPE foo [ <!ELEMENT foo ANY >
+    <!ENTITY xxe SYSTEM \"file:///etc/passwd\" >]>
+    <response><result>&xxe;</result></response>
+    """
+
+    _ = Process.flag(:trap_exit, true)
+    pid = spawn_link(fn -> Stream.run(SweetXml.stream_tags(sneaky_xml, :banana, dtd: :none, quiet: true)) end)
+    assert_receive {:EXIT, ^pid, {:fatal, {{:error_fetching_DTD, {_, _}}, _file, _line, _col}}}
+  end
+
+  test "stream: raise on billion_laugh.xml with dtd: :none" do
+    dangerous_xml = File.read!("./test/files/billion_laugh.xml")
+
+    _ = Process.flag(:trap_exit, true)
+    pid = spawn_link(fn -> Stream.run(SweetXml.stream_tags(dangerous_xml, :banana, dtd: :none, quiet: true)) end)
+    assert_receive {:EXIT, ^pid, {%RuntimeError{}, _stacktrace}}
+  end
 end