sys_core_fold: Eliminate crash in beam_kernel_to_ssa, again

Author: jhogberg

lib/compiler/src/sys_core_fold.erl

@@ -2216,32 +2216,24 @@ move_let_into_expr(#c_let{vars=InnerVs0,body=InnerBody0}=Inner0,
 
     OuterBody = body(OuterBody0, ScopeSub),
 
-    {InnerVs,Sub} = var_list(InnerVs0, Sub0),
-    InnerBody = body(InnerBody0, Sub),
-    Inner = case will_fail(OuterBody) of
-                true ->
-                    %%
-                    %% Avoid creating potentially unsafe code that
-                    %% depends on another iteration of the outer
-                    %% fixpoint loop to clean up. If <OuterBody>
-                    %% consists of nested lets, we may run out of
-                    %% iterations before the unsafe code is
-                    %% eliminated.
-                    %%
-                    %% let <InnerVars> = let <OuterVars> = <Arg>
-                    %%                   in <OuterBody>
-                    %% in <InnerBody>
-                    %%
-                    %%       ==>
-                    %%
-                    %% let <OuterVars> = <Arg>
-                    %% in <OuterBody>
-                    %%
-                    OuterBody;
-                false ->
-                    Inner0#c_let{vars=InnerVs,arg=OuterBody,body=InnerBody}
-            end,
-    Outer#c_let{vars=OuterVs,arg=Arg,body=Inner};
+    case will_fail(OuterBody) of
+        true ->
+            %%
+            %% Avoid creating potentially unsafe code that
+            %% depends on another iteration of the outer
+            %% fixpoint loop to clean up. If <OuterBody>
+            %% consists of nested lets, we may run out of
+            %% iterations before the unsafe code is
+            %% eliminated.
+            %%
+            Inner0;
+        false ->
+            {InnerVs,Sub} = var_list(InnerVs0, Sub0),
+            InnerBody = body(InnerBody0, Sub),
+        
+            Inner = Inner0#c_let{vars=InnerVs,arg=OuterBody,body=InnerBody},
+            Outer#c_let{vars=OuterVs,arg=Arg,body=Inner}
+    end;
 move_let_into_expr(#c_let{vars=Lvs0,body=Lbody0}=Let,
 		   #c_case{arg=Cexpr0,clauses=[Ca0|Cs0]}=Case, Sub0) ->
     case not is_failing_clause(Ca0) andalso

lib/compiler/test/core_fold_SUITE.erl

@@ -701,41 +701,98 @@ do_receive_effect() ->
     {} = receive _ -> {} = {} end.
 
 nested_lets(_Config) ->
-    {'EXIT',{{case_clause,ok},_}} = catch nested_lets_gh_6572(<<42>>),
+    {'EXIT',{{case_clause,ok},_}} = catch nested_lets_1(<<42>>),
+    {'EXIT',{badarith,_}} = catch nested_lets_2(id(0), id(0)),
     ok.
 
-nested_lets_gh_6572(<<X>>) ->
+%% GH-6572: Deeply nested `let` expressions caused `sys_core_fold` to generate
+%% unsafe code that it would attempt to fix up later. Unfortunately it did so
+%% through a limited fixpoint iteration, and would leak said code once the
+%% limit was hit.
+nested_lets_1(<<X>>) ->
     Y =
         case ok of
             X ->
-                true = (ok > ((Y = _) = -1)),
+                true = (ok > (Y = -1)),
                 <<>> =
                     {id(
-                        <<
-                            (ok - ok),
-                            (bnot ok),
-                            (nested_lets_gh_6572_f() band ok),
-                            (nested_lets_gh_6572_f()),
-                            (not ok),
-                            (ok or nested_lets_gh_6572_f()),
-                            (id(
-                                id(
-                                    <<
-                                        (id(
-                                            <<
-                                                (id(
-                                                    <<0 || _ <- []>>
-                                                ))
-                                            >>
-                                        ) * ok)
-                                    >>
-                                )
-                            ))
-                        >>
-                    )}
+                       <<
+                         (ok - ok),
+                         (bnot ok),
+                         (nested_lets_1_f() band ok),
+                         (nested_lets_1_f()),
+                         (not ok),
+                         (ok or nested_lets_1_f()),
+                         (id(
+                            id(
+                              <<
+                                (id(
+                                   <<
+                                     (id(
+                                        <<0 || _ <- []>>
+                                       ))
+                                   >>
+                                  ) * ok)
+                              >>
+                             )
+                           ))
+                       >>
+                      )}
         end.
 
-nested_lets_gh_6572_f() ->
+nested_lets_1_f() ->
+    ok.
+
+%% GH-6612: A variant of GH-6572 that slipped through the initial fix.
+nested_lets_2(X, 0) ->
+    try
+        0 = {
+             _ = 0 + 0,
+             Z = bnot ok,
+             {(_ = ok), (_ = X)}#{ok => ok},
+             ok +
+                 (nested_lets_2_f(
+                    ok +
+                        nested_lets_2_f(
+                          ok +
+                              (nested_lets_2_f(
+                                                (Y =
+                                                     -nested_lets_2_f(
+                                                        #{
+                                                            (ok +
+                                                                 (ok +
+                                                                      nested_lets_2_f(
+                                                                        case
+                                                                            try ok of
+                                                                                _ ->
+                                                                                    fun(_) ->
+                                                                                            ok
+                                                                                    end
+                                                                            after
+                                                                                ok
+                                                                            end
+                                                                        of
+                                                                            #{} ->
+                                                                                ok
+                                                                        end
+                                                                       ))) => ok
+                                                         } > ok
+                                                       ))
+                                ) + ok)
+                         ) >
+                        ok
+                   ))
+            }
+    of
+        _ ->
+            Z;
+        _ ->
+            Y
+    after
+        ok
+    end.
+
+nested_lets_2_f(_) ->
     ok.
 
 %%% Common utility functions.