Add option to PULSE test to create merge.txt file on bc_open()

Fix PULSE counterexample by filtering error messages.

    C1 = [[{set,{var,6},
           {call,bitcask_pulse,fork,
                 [[{init,{state,undefined,false,false,[]}},
                   {set,{not_var,7},
                        {not_call,bitcask_pulse,bc_open,
                                  [false,{true,{231,280,114},1}]}},
                   {set,{not_var,10},
                        {not_call,bitcask_pulse,bc_close,[{not_var,7}]}},
                   {set,{not_var,11},
                        {not_call,bitcask_pulse,bc_open,
                                  [false,{true,{170,447,173},1}]}},
                   {set,{not_var,13},
                        {not_call,bitcask_pulse,bc_close,[{not_var,11}]}},
                   {set,{not_var,15},
                        {not_call,bitcask_pulse,bc_open,
                                  [false,{true,{258,18,280},1}]}}]]}},
      {set,{var,7},
           {call,bitcask_pulse,fork,
                 [[{init,{state,undefined,false,false,[]}},
                   {set,{not_var,2},
                        {not_call,bitcask_pulse,bc_open,
                                  [false,{true,{246,472,469},1}]}},
                   {set,{not_var,3},
                        {not_call,bitcask_pulse,bc_close,[{not_var,2}]}},
                   {set,{not_var,6},
                        {not_call,bitcask_pulse,bc_open,
                                  [false,{true,{73,166,183},1}]}},
                   {set,{not_var,7},
                        {not_call,bitcask_pulse,fold,[{not_var,6}]}}]]}},
      {set,{var,8},
           {call,bitcask_pulse,bc_open,[true,{true,{424,237,233},1}]}},
      {set,{var,10},{call,bitcask_pulse,needs_merge,[{var,8}]}},
      {set,{var,11},{call,bitcask_pulse,merge,[{var,8}]}},
      {set,{var,12},{call,bitcask_pulse,delete,[{var,8},1]}},
      {set,{var,13},{call,bitcask_pulse,merge,[{var,8}]}}],
     {62806,53384,84006},
     [{errors,[]}]].
    eqc:check(bitcask_pulse:prop_pulse(), C1).

The error coming from error_logger & failing the property:

    errors: failed
    [{<0.23310.2>,"Invalid merge input file ~s, deleting : ~p",
      ["1399-272921-941361/merge.txt",enoent]}] /= []

This error message is not fatal, so ignore it.

Fix PULSE counterexample when merge's ReadableFiles is the empty list

    C2 = [[{set,{var,2},
           {call,bitcask_pulse,bc_open,[true,{true,{442,453,173},1}]}},
      {set,{var,3},{call,bitcask_pulse,delete,[{var,2},1]}},
      {set,{var,5},{call,bitcask_pulse,bc_close,[{var,2}]}},
      {set,{var,8},
           {call,bitcask_pulse,bc_open,
                 [true,{true,{172,181,201},100}]}},
      {set,{var,9},{call,bitcask_pulse,fork_merge,[{var,8}]}},
      {set,{var,11},{call,bitcask_pulse,fork_merge,[{var,8}]}}],
     {41544,86022,14862},
     [{errors,[]}]].
    eqc:check(bitcask_pulse:prop_pulse(), C2).

The error from PULSE:

    errors: failed
    [{<0.27136.3>,"Failed to merge ~p: ~p\n",
      [{["1399-274027-165496/1.bitcask.data"],[]},
       {function_clause,
           [{lists,min,[[]],[{file,"lists.erl"},{line,300}]},
            {bitcask,merge1,4,[{file,"src/bitcask.erl"},{line,611}]},
            {bitcask_merge_worker,do_merge,1,
                [{file,"src/bitcask_merge_worker.erl"},{line,195}]}]}]}] /= []

Fix for Buildbot EUnit test failure

Buildbot isn't quite right in the head: I've no idea why I can
run this test 3000x without failure, but Buildbot fails

    bitcask_qc: merge2_test...
    =ERROR REPORT==== 5-May-2014::04:56:18 ===
    ** Generic server <0.2324.0> terminating
    ** Last message in was {'DOWN',#Ref<0.0.0.50927>,process,<0.2317.0>,normal}
    ** When Server state == {state,undefined,undefined}
    ** Reason for termination ==
    ** {{badmatch,{error,badarg}},
        [{bitcask_file,handle_info,2,[{file,"src/bitcask_file.erl"},{line,193}]},
         {gen_server,handle_msg,5,[{file,"gen_server.erl"},{line,604}]},
         {proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,239}]}]}

Fix: Really, we're stopping, so change the pattern match to '_' so we
don't care.

Fix Dialyzer type typo

Fix Buildbot timeout error

      bitcask: merge_batch_test...*timed out*

Increase timeout to something even a Buildbot can love.

Author: slfritchie

src/bitcask.erl

@@ -608,8 +608,12 @@ merge1(Dirname, Opts, FilesToMerge0, ExpiredFiles) ->
                          end, InFiles2),
     InFileIds = sets:from_list([bitcask_fileops:file_tstamp(InFile)
                                || InFile <- InFiles]),
-    MinFileId = lists:min([bitcask_fileops:file_tstamp(F) ||
-                           F <- ReadableFiles]),
+    MinFileId = if ReadableFiles == [] ->
+                        1;
+                   true ->
+                        lists:min([bitcask_fileops:file_tstamp(F) ||
+                                      F <- ReadableFiles])
+                end,
 
     %% Initialize the other keydirs we need.
     {ok, DelKeyDir} = bitcask_nifs:keydir_new(),
@@ -679,7 +683,7 @@ consider_for_merge(FragTrigger, DeadBytesTrigger, ExpirationGraceTime) ->
 needs_merge(Ref) ->
     needs_merge(Ref, []).
 
--spec needs_merge(reference(), proplist:proplist()) -> {true, {[string()], [string()]}} | false.
+-spec needs_merge(reference(), proplists:proplist()) -> {true, {[string()], [string()]}} | false.
 needs_merge(Ref, Opts) ->
     State = get_state(Ref),
     {_KeyCount, Summary} = summary_info(Ref),
@@ -2977,7 +2981,10 @@ total_byte_stats_test() ->
     ?assertEqual(ExpFiles1, Files1),
     bitcask:close(B).
 
-merge_batch_test() ->
+merge_batch_test_() ->
+    {timeout, 100, fun merge_batch_test2/0}.
+
+merge_batch_test2() ->
     Dir = "/tmp/bc.merge.batch",
     % Create a valid Bitcask dir with files 10-20 present only
     DataSet = [{integer_to_binary(N), <<"data">>} || N <- lists:seq(1,20)],

src/bitcask_file.erl

@@ -190,7 +190,7 @@ handle_cast(_Msg, State) ->
 
 handle_info({'DOWN', _Ref, _, _Pid, _Status}, State=#state{fd=Fd}) ->
     %% Owner has stopped, close file and shutdown
-    ok = file:close(Fd),
+    _ = file:close(Fd),
     {stop, normal, State};
 handle_info(_Info, State) ->
     {noreply, State}.

test/bitcask_pulse.erl

@@ -69,6 +69,15 @@ not_commands(Module, State) ->
   ?LET(Cmds, commands(Module, State),
     uncommand(Cmds)).
 
+gen_make_merge_file() ->
+    %% {true|false, Seed, Probability}
+    {
+      frequency([{100, false}, {109999, true}]),
+      noshrink({choose(0, 500), choose(0, 500), choose(0, 500)}),
+      %% weighting: use either 100% or 1-100%
+      frequency([{2, 100}, {8, choose(1, 100)}])
+    }.
+
 %% Command generator. S is the state.
 command(S) ->
   frequency(
@@ -94,7 +103,7 @@ command(S) ->
     %% race bugs, so keep doing it.
     [ {50, {call, ?MODULE, delete, [S#state.handle, key()]}}
       || S#state.is_writer, S#state.handle /= undefined ] ++
-    [ {2, {call, ?MODULE, bc_open, [S#state.is_writer]}}
+    [ {2, {call, ?MODULE, bc_open, [S#state.is_writer, gen_make_merge_file()]}}
       || S#state.handle == undefined ] ++
     [ {2, {call, ?MODULE, sync, [S#state.handle]}}
       || S#state.handle /= undefined ] ++
@@ -156,7 +165,7 @@ precondition(S, {call, _, kill, [Pid]}) ->
   (Pid == bitcask_merge_worker orelse lists:member(Pid, S#state.readers));
 precondition(S, {call, _, bc_close, [H]}) ->
   S#state.handle == H;
-precondition(S, {call, _, bc_open, [Writer]}) ->
+precondition(S, {call, _, bc_open, [Writer, _MakeMergeFile]}) ->
   %% The writer can open for reading but not the other way around.
   S#state.is_writer >= Writer andalso S#state.handle == undefined.
 
@@ -223,7 +232,7 @@ postcondition(_S, {call, _, needs_merge, _}, V) ->
     false          -> true;
     _              -> {needs_merge, V}
   end;
-postcondition(_S, {call, _, bc_open, [IsWriter]}, V) ->
+postcondition(_S, {call, _, bc_open, [IsWriter, _MakeMergeFile]}, V) ->
   case V of
     _ when is_reference(V) andalso IsWriter -> check_no_tombstones(V, true);
     _ when is_reference(V)                  -> true;
@@ -361,7 +370,9 @@ prop_pulse(LocalOrSlave, Verbose, KeepFiles) ->
       {badrpc, timeout} = Bad ->
         io:format(user, "GOT ~p, aborting.  Stop PULSE and restart!\n", [Bad]),
         exit({stopping, Bad});
-      {H, S, Res, PidRs, Trace, Schedule, Errors} ->
+      {H, S, Res, PidRs, Trace, Schedule, Errors0} ->
+        Errors = [E || E <- Errors0,
+                       re:run(element(2, E), "Invalid merge input") == nomatch],
         ?WHENFAIL(
           ?QC_FMT("\nState: ~p\n", [S]),
           aggregate(zipwith(fun command_data/2, Cmds, H),
@@ -422,7 +433,7 @@ prop_pulse_test_() ->
     end,
     io:format(user, "prop_pulse_test time: ~p + ~p seconds\n",
               [Timeout, ExtraTO]),
-    {timeout, (Timeout+ExtraTO),
+    {timeout, (Timeout+ExtraTO+30),             % 30 = a bit more fudge time
      fun() ->
              copy_bitcask_app(),
              ?assert(eqc:quickcheck(eqc:testing_time(Timeout,
@@ -785,8 +796,13 @@ fold_keys(H) ->
   ?LOG({fold_keys, H},
   ?CHECK_HANDLE(H, [], bitcask:fold_keys(H, fun(#bitcask_entry{key = Kb}, Ks) -> [un_nice_key(Kb)|Ks] end, []))).
 
-bc_open(Writer) ->
+bc_open(Writer, {MakeMergeFileP, Seed, Probability}) ->
   erlang:put(?BITCASK_TESTING_KEY, ?MODULE),
+    if MakeMergeFileP ->
+            make_merge_file(?BITCASK, Seed, Probability);
+       true ->
+            ok
+    end,
   ?LOG({open, Writer},
   case Writer of
     true  -> catch bitcask:open(?BITCASK, [read_write, {max_file_size, ?FILE_SIZE}, {open_timeout, 1234}]);
@@ -1091,4 +1107,21 @@ check_no_tombstones(Ref, Good) ->
             {check_no_tombstones, Else}
     end.
 
+make_merge_file(Dir, Seed, Probability) ->
+    random:seed(Seed),
+    case filelib:is_dir(Dir) of
+        true ->
+            DataFiles = filelib:wildcard("*.data", Dir),
+            {ok, FH} = file:open(Dir ++ "/merge.txt", [write]),
+            [case random:uniform(100) < Probability of
+                 true ->
+                     io:format(FH, "~s\n", [DF]);
+                 false ->
+                     ok
+             end || DF <- DataFiles],
+            ok = file:close(FH);
+        false ->
+            ok
+    end.
+
 -endif.