diff --git a/Gemfile.lock b/Gemfile.lock index cb4296e..29175db 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -13,7 +13,7 @@ GEM pry (0.11.3) coderay (~> 1.1.0) method_source (~> 0.9.0) - rake (12.3.1) + rake (12.3.3) PLATFORMS ruby diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..a957cad --- /dev/null +++ b/LICENSE @@ -0,0 +1,169 @@ +This implementation of Magritte is provided under the terms of the GNU LGPL v3, which is provided below. +See https://opensource.org/licenses/LGPL-3.0 + + + GNU LESSER GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + + This version of the GNU Lesser General Public License incorporates +the terms and conditions of version 3 of the GNU General Public +License, supplemented by the additional permissions listed below. + + 0. Additional Definitions. + + As used herein, "this License" refers to version 3 of the GNU Lesser +General Public License, and the "GNU GPL" refers to version 3 of the GNU +General Public License. + + "The Library" refers to a covered work governed by this License, +other than an Application or a Combined Work as defined below. + + An "Application" is any work that makes use of an interface provided +by the Library, but which is not otherwise based on the Library. +Defining a subclass of a class defined by the Library is deemed a mode +of using an interface provided by the Library. + + A "Combined Work" is a work produced by combining or linking an +Application with the Library. The particular version of the Library +with which the Combined Work was made is also called the "Linked +Version". + + The "Minimal Corresponding Source" for a Combined Work means the +Corresponding Source for the Combined Work, excluding any source code +for portions of the Combined Work that, considered in isolation, are +based on the Application, and not on the Linked Version. + + The "Corresponding Application Code" for a Combined Work means the +object code and/or source code for the Application, including any data +and utility programs needed for reproducing the Combined Work from the +Application, but excluding the System Libraries of the Combined Work. + + 1. Exception to Section 3 of the GNU GPL. + + You may convey a covered work under sections 3 and 4 of this License +without being bound by section 3 of the GNU GPL. + + 2. Conveying Modified Versions. + + If you modify a copy of the Library, and, in your modifications, a +facility refers to a function or data to be supplied by an Application +that uses the facility (other than as an argument passed when the +facility is invoked), then you may convey a copy of the modified +version: + + a) under this License, provided that you make a good faith effort to + ensure that, in the event an Application does not supply the + function or data, the facility still operates, and performs + whatever part of its purpose remains meaningful, or + + b) under the GNU GPL, with none of the additional permissions of + this License applicable to that copy. + + 3. Object Code Incorporating Material from Library Header Files. + + The object code form of an Application may incorporate material from +a header file that is part of the Library. You may convey such object +code under terms of your choice, provided that, if the incorporated +material is not limited to numerical parameters, data structure +layouts and accessors, or small macros, inline functions and templates +(ten or fewer lines in length), you do both of the following: + + a) Give prominent notice with each copy of the object code that the + Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the object code with a copy of the GNU GPL and this license + document. + + 4. Combined Works. + + You may convey a Combined Work under terms of your choice that, +taken together, effectively do not restrict modification of the +portions of the Library contained in the Combined Work and reverse +engineering for debugging such modifications, if you also do each of +the following: + + a) Give prominent notice with each copy of the Combined Work that + the Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the Combined Work with a copy of the GNU GPL and this license + document. + + c) For a Combined Work that displays copyright notices during + execution, include the copyright notice for the Library among + these notices, as well as a reference directing the user to the + copies of the GNU GPL and this license document. + + d) Do one of the following: + + 0) Convey the Minimal Corresponding Source under the terms of this + License, and the Corresponding Application Code in a form + suitable for, and under terms that permit, the user to + recombine or relink the Application with a modified version of + the Linked Version to produce a modified Combined Work, in the + manner specified by section 6 of the GNU GPL for conveying + Corresponding Source. + + 1) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (a) uses at run time + a copy of the Library already present on the user's computer + system, and (b) will operate properly with a modified version + of the Library that is interface-compatible with the Linked + Version. + + e) Provide Installation Information, but only if you would otherwise + be required to provide such information under section 6 of the + GNU GPL, and only to the extent that such information is + necessary to install and execute a modified version of the + Combined Work produced by recombining or relinking the + Application with a modified version of the Linked Version. (If + you use option 4d0, the Installation Information must accompany + the Minimal Corresponding Source and Corresponding Application + Code. If you use option 4d1, you must provide the Installation + Information in the manner specified by section 6 of the GNU GPL + for conveying Corresponding Source.) + + 5. Combined Libraries. + + You may place library facilities that are a work based on the +Library side by side in a single library together with other library +facilities that are not Applications and are not covered by this +License, and convey such a combined library under terms of your +choice, if you do both of the following: + + a) Accompany the combined library with a copy of the same work based + on the Library, uncombined with any other library facilities, + conveyed under the terms of this License. + + b) Give prominent notice with the combined library that part of it + is a work based on the Library, and explaining where to find the + accompanying uncombined form of the same work. + + 6. Revised Versions of the GNU Lesser General Public License. + + The Free Software Foundation may publish revised and/or new versions +of the GNU Lesser General Public License from time to time. Such new +versions will be similar in spirit to the present version, but may +differ in detail to address new problems or concerns. + + Each version is given a distinguishing version number. If the +Library as you received it specifies that a certain numbered version +of the GNU Lesser General Public License "or any later version" +applies to it, you have the option of following the terms and +conditions either of that published version or of any later version +published by the Free Software Foundation. If the Library as you +received it does not specify a version number of the GNU Lesser +General Public License, you may choose any version of the GNU Lesser +General Public License ever published by the Free Software Foundation. + + If the Library as you received it specifies that a proxy can decide +whether future versions of the GNU Lesser General Public License shall +apply, that proxy's public statement of acceptance of any version is +permanent authorization for you to choose that version for the +Library. diff --git a/README.md b/README.md new file mode 100644 index 0000000..ede0d7a --- /dev/null +++ b/README.md @@ -0,0 +1,5 @@ +# Magritte, a language for pipe-based programming + +This is a reference implementation of the Magritte language, described in @jneen's thesis: http://files.jneen.net/academic/thesis.pdf + +The interpreter is at `./bin/mag`. diff --git a/example/basic.mag b/example/basic.mag index 3060a65..b0de65e 100644 --- a/example/basic.mag +++ b/example/basic.mag @@ -1 +1,4 @@ -put 1 2 3 +x = 1 +# put $x + +(f) = (put 10) diff --git a/example/prime.mag b/example/prime.mag index 6af5747..21e841a 100644 --- a/example/prime.mag +++ b/example/prime.mag @@ -2,13 +2,13 @@ (binary-rand) = (round 0 (rand)) # Generate a sequence of n random bits -(gen-bits ?n) = (produce binary-rand | take %n) +(gen-bits ?n) = (produce %binary-rand | take %n) # Convert a number in base b to base 10 (from-base ?b) = ( num = 0 each (?d => - %num = (mul %b %num | into add %d) + %num = (mul %b %num | into %add %d) ) put %num ) @@ -17,7 +17,10 @@ # breaks when n < 3 because we want to ensure that the leading and # trailing bit is set to 1. Of course, nobody should actually call the # function with n < 3 because that defeats the purpose of the function. -(gen-number ?n) = ((put 1; (gen-bits (add %n -2)); put 1) | from-base 2) +(gen-number ?n) = ( + (put 1; (gen-bits (add %n -2)); put 1) + | from-base 2 +) # In Miller-rabin we want to factor a number num = (2^r)*d # and learn what r and d is. This function returns r and d @@ -53,16 +56,35 @@ # You can increase i to increase the certainty. # Fails if num is not prime (miller-rabin ?i ?num) = ( - eq 1 (mod 2 %num) && - (r d = (miller-rabin-factoring (dec %num) 0) - (miller-rabin-test %i %num %r %d)) + eq 1 (mod 2 %num) && ( + r d = (miller-rabin-factoring (dec %num) 0) + (miller-rabin-test %i %num %r %d) + ) ) -(repeat-func ?fn) = (put (exec %fn); repeat-func %fn) +(gen-primes ?b) = ( + # produce (=> gen-number %b) | take 10 + produce [%gen-number %b] | each (?n => iter %inc %n | take %b) + | filter [%miller-rabin 10] +) + +(gen-prime ?b) = (gen-primes %b | take 1) + +(sum) = ( + num = 0 + each (?d => + %num = (add %d %num) # | into %add %d) + ) + put %num +) -(gen-prime ?b) = ( - repeat-func [gen-number %b] | each (?n => iter (?x => inc %x) %n | take %b) | - filter [miller-rabin 10] | take 1 +(testy) = ( + produce (=> + put 1 | sum + ) | take 10 + put done ) -put (gen-prime 5) +(__main__) = gen-primes 10 +# (__main__) = testy +# (__main__) = put 1 diff --git a/example/server.mag b/example/server.mag new file mode 100644 index 0000000..a9c8c80 --- /dev/null +++ b/example/server.mag @@ -0,0 +1,49 @@ +(spawn (?fn)) = ( + i = (make-channel) + o = (make-channel) + & exec (for %fn) < $i > $o + log after internal spawn + put $i $o + log returning from spawn +) + +(server (?fn)) = ( + log server + i o = (spawn (=> + log spawned (for %fn) + each ([?msg ?ch] => + log received [%msg %ch] + exec (for %fn) %msg > %ch + ) + )) + + log after spawn + + # keep the channel alive on the read end + & sleep-forever > $i + + put { ch = %i } +) + +(send ?server ?msg) = ( + log send %server %msg + c = (make-channel) + o = %server!ch + put [%msg $c] > $o + log after-send + drain < $c +) + +(__main__) = ( + log hello + s = (server ( + [greet ?x] => str "hello " %x + [die] => crash + )) + + log after server + + send %s [greet world] + log greeted + send %s [die] +) diff --git a/lib/magritte/ast.rb b/lib/magritte/ast.rb index 49818ee..2ba4603 100644 --- a/lib/magritte/ast.rb +++ b/lib/magritte/ast.rb @@ -1,40 +1,56 @@ module Magritte module AST - class Variable < Tree::Node defdata :name + + def repr; "$#{name}"; end end class LexVariable < Tree::Node defdata :name + + def repr; "%#{name}"; end end class Binder < Tree::Node defdata :name + + def repr; "?#{name}"; end end class String < Tree::Node defdata :value + + def repr; value.inspect; end end class Number < Tree::Node defdata :value + + def repr; value.inspect; end end class StringPattern < Tree::Node defdata :value + + def repr; "~#{value.inspect}"; end end class VectorPattern < Tree::Node deflistrec :patterns defopt :rest + + def repr; "~[#{(patterns + [rest].compact).map(&:repr).join(' ')}]"; end end class DefaultPattern < Tree::Node + def repr; "_"; end end class RestPattern < Tree::Node defrec :binder + + def repr; "*#{binder.repr}"; end end class Lambda < Tree::Node @@ -47,11 +63,22 @@ def initialize(*) super raise "Pattern and body mismatch" unless patterns.size == bodies.size end + + def repr + out = "(" + out << patterns.zip(bodies).map do |(pat, bod)| + "#{pat.repr} => #{bod.repr}" + end.join('; ') + out << ')' + out + end end class Pipe < Tree::Node defrec :producer defrec :consumer + + def repr; "#{producer.repr} | #{consumer.repr}"; end end class Or < Tree::Node @@ -61,6 +88,8 @@ class Or < Tree::Node def continue?(status) status.fail? end + + def repr; "#{lhs.parrepr} || #{rhs.parrepr}"; end end class And < Tree::Node @@ -70,11 +99,15 @@ class And < Tree::Node def continue?(status) status.normal? end + + def repr; "#{lhs.parrepr} && #{rhs.parrepr}"; end end class Else < Tree::Node defrec :lhs defrec :rhs + + def repr; "#{lhs.parrepr} !! #{rhs.parrepr}"; end end class Compensation < Tree::Node @@ -82,20 +115,36 @@ class Compensation < Tree::Node defrec :compensation defdata :range defdata :unconditional + + def repr + "#{expr.repr} #{unconditional ? '%%!' : '%%'} #{compensation.parrepr}" + end end class Spawn < Tree::Node defrec :expr + + def repr + "& #{expr.repr}" + end end class Redirect < Tree::Node defdata :direction defrec :target + + def repr + "#{direction} #{target.repr}" + end end class With < Tree::Node deflistrec :redirects defrec :expr + + def repr + "#{expr.repr} #{redirects.map(&:repr).join(' ')}" + end end class Command < Tree::Node @@ -106,36 +155,59 @@ def initialize(*) super raise "Empty command" unless vec.any? end + + def repr; vec.map(&:repr).join(' '); end end class Block < Tree::Node defrec :group + + def repr; "(@block #{group.repr})"; end end class Group < Tree::Node deflistrec :elems + + def repr; elems.map(&:repr).join('; '); end end class Subst < Tree::Node defrec :group + + def repr; "(#{group.repr})"; end end class Vector < Tree::Node deflistrec :elems + + def repr; "[#{elems.map(&:repr).join(' ')}]"; end end class Environment < Tree::Node defrec :body + + def repr; "{ #{body.repr} }"; end end class Access < Tree::Node defrec :source defrec :lookup + + def repr; "#{source.parrepr}!#{lookup.repr}"; end end class Assignment < Tree::Node deflistrec :lhs deflistrec :rhs + + def repr; "#{lhs.map(&method(:lhs_repr)).join(' ')} = #{rhs.map(&:repr).join(' ')}"; end + + private + def lhs_repr(n) + return n.value if n.is_a? AST::String + + n.repr + end end end end diff --git a/lib/magritte/builtins.rb b/lib/magritte/builtins.rb index 7543127..29dceed 100644 --- a/lib/magritte/builtins.rb +++ b/lib/magritte/builtins.rb @@ -9,6 +9,8 @@ def self.load(env) env.let(name, func) end load_file("#{ROOT_DIR}/mag/prelude.mag", env) + + env end @builtins = [] @@ -40,6 +42,19 @@ def self.builtin(name, arg_types, rest_type = nil, &impl) Status.normal end + builtin :'load-raw', [:String] do |fname| + Builtins.load_file(fname.value, Proc.current.env) + end + + builtin :load, [:String] do |fname| + status = nil + env = in_new_env(Proc.current.env) do + status = Builtins.load_file(fname.value, Proc.current.env) + end + put Value::Environment.new(env) + status + end + builtin :get, [] do put(get) Status.normal @@ -88,6 +103,10 @@ def self.builtin(name, arg_types, rest_type = nil, &impl) Status.normal end + builtin :'sleep-forever', [] do + sleep + end + builtin :'count-forever', [] do |n| i = 0 produce { put Value::Number.new(i); i += 1 } @@ -101,15 +120,17 @@ def self.builtin(name, arg_types, rest_type = nil, &impl) builtin :'loop-channel', [:Channel, :any], :any do |c, h, *a| channel = c.channel - loop_channel(c.channel) { call(h, a) } + loop_channel(c.channel) { PRINTER.p("XXX"); call(h, a); PRINTER.p("YYY") } end builtin :stdin, [] do + PRINTER.p :stdin => Proc.current.env.stdin + put Value::Channel.new(Proc.current.stdin) end builtin :stdout, [] do - put Value::Channel.new(Proc.current.stdout) + put Value::Channel.new(Proc.current.env.up.stdout || Proc.current.stdout) end builtin :add, [], :Number do |*nums| @@ -273,14 +294,8 @@ def self.builtin(name, arg_types, rest_type = nil, &impl) # Initialize environment with functions that can be defined in the language itself def self.load_lib(lib_name, source, env) - # Transform the lib string into an ast ast = Parser.parse(Skeleton.parse(Lexer.new(lib_name, source))) - c = Collector.new - env.own_outputs[0] = c - # Evaluate the ast, which will add the lib functions to the env - Proc.spawn(Code.new { Interpret.interpret(ast) }, env).start - c.wait_for_close - env + Proc.spawn(Code.new { Interpret.interpret(ast) }, env).wait end def self.load_file(file_path, env) diff --git a/lib/magritte/channel.rb b/lib/magritte/channel.rb index e162b5e..c03f8f5 100644 --- a/lib/magritte/channel.rb +++ b/lib/magritte/channel.rb @@ -1,3 +1,5 @@ +$DEBUG_MUTEX = Mutex.new + module Magritte class Blocker attr_reader :thread, :val @@ -40,18 +42,29 @@ def disp end def log(*a) - # PRINTER.p disp => a + if PRINTER.is_a?(LogPrinter) + File.open("tmp/log/#{$$}/#{disp}", 'a') { |f| f.puts *a } + else + PRINTER.puts(*a) + end + end + + def p(*a) + log(disp, *a.map(&:inspect)) end def synchronize(&b) + trace = caller[0] + out = nil - log :lock @mutex.synchronize do - log :locked - out = yield - log :unlock + begin + # log "#{disp} lock : #{trace}" + out = yield + ensure + # log "#{disp} unlock: #{trace}" + end end - log :unlocked out end @@ -60,21 +73,28 @@ def owned? end def sleep - log :sleep + log "#{disp} sleep : #{caller[0]}" @mutex.sleep - log :wake + log "#{disp} wake" end end + ALL_CHANNELS = [] class Channel IDS_MUTEX = Mutex.new @@max_id = 0 def initialize - @mutex = Mutex.new + setup_id setup - @id = IDS_MUTEX.synchronize { @@max_id += 1 } + end + def setup_id + IDS_MUTEX.synchronize do + @id = @@max_id += 1 + @mutex = LogMutex.new("#{self.class.name.downcase}_#{@id}") + ALL_CHANNELS << self + end end def setup @@ -87,9 +107,9 @@ def setup end def reset! - @mutex.synchronize { @open = false } - interrupt_blocked! - @mutex.synchronize { setup } + # @mutex.synchronize { @open = false } + # interrupt_blocked! + # @mutex.synchronize { setup } end def open? @@ -110,45 +130,69 @@ def add_writer(p) def remove_reader(p) action = @mutex.synchronize do + @mutex.log "[#{@id}] remove_reader #{@readers.size} #{p.inspect}" + next :nop unless @open @block_set.reject! { |b| b.thread == p.thread } if @block_type == :read @readers.delete(p) + @mutex.p([@block_type, @block_set.size]) next :nop unless @readers.empty? + @mutex.log "[#{@id}] CLOSE: #{@block_set.map(&:inspect).join(' ')}" + @open = false :close end - PRINTER.p(closing_channel: @id) if action == :close - interrupt_blocked! if action == :close + if action == :close + PRINTER.p(close: [@id, @block_type, @block_set]) + end + + interrupt_blocked!(:>) if action == :close + rescue ThreadError + binding.pry end def remove_writer(p) action = @mutex.synchronize do next :nop unless open? + @mutex.log "[#{@id}] remove_writer #{@writers.size} #{p.inspect}" @block_set.reject! { |b| b.thread == p.thread } if @block_type == :write @writers.delete(p) next :nop unless @writers.empty? + @mutex.log "[#{@id}] CLOSE: #{@block_set.map(&:inspect).join(' ')}" @open = false :close end PRINTER.p(closing_channel: @id) if action == :close - interrupt_blocked! if action == :close + interrupt_blocked!(:<) if action == :close end - def interrupt_blocked! - @block_set.each { |b| interrupt_process!(b) } + def interrupt_blocked!(d) + interrupt_self = false + @block_set.each do |b| + if b.thread == Thread.current + interrupt_self = true + else + interrupt_process!(b, d) + end + end + + interrupt_process!(Proc.current, d) if interrupt_self end def read - @mutex.synchronize do - if closed? - PRINTER.p("closed on read") + result = @mutex.synchronize do + unless @open + PRINTER.p("#{@id} read: already closed") + @mutex.p("interrupting #{LogPrinter.thread_name(Thread.current)}") + + interrupt_process!(Proc.current, :<) # jump to the end of the block next @@ -156,7 +200,8 @@ def read @block_set.shuffle! - PRINTER.p(init_read: @block_type) + @mutex.p(read: [@id, @block_type, open: @open]) + PRINTER.p(read: [@id, @block_type, open: @open]) out = case @block_type when :none, :read @block_type = :read @@ -164,63 +209,63 @@ def read receiver = Receiver.new(Thread.current) @block_set << receiver @mutex.sleep - - return receiver.val + @block_set.delete(receiver) + receiver when :write sender = @block_set.shift @block_type = :none if @block_set.empty? sender.wakeup - return sender.val + sender end end - interrupt_process!(Proc.current) + Proc.check_interrupt! + + result.val end def write(val) @mutex.synchronize do - if closed? - PRINTER.p("closed on write") + unless @open + PRINTER.p("#{@id} write: already closed") + @mutex.p("interrupting #{LogPrinter.thread_name(Thread.current)}") + interrupt_process!(Proc.current, :>) next end @block_set.shuffle! - PRINTER.p(init_write: @block_type) + @mutex.p(write: [@id, @block_type]) + PRINTER.p(write: [@id, @block_type]) case @block_type when :none, :write @block_type = :write - @block_set << Sender.new(Thread.current, val) + sender = Sender.new(Thread.current, val) + @block_set << sender @mutex.sleep - return + @block_set.delete(sender) when :read - receiver = @block_set.shift + receiver = @block_set.shift # or PRINTER.p(EMPTY_BLOCK_SET: [@block_type, self]) @block_type = :none if @block_set.empty? receiver.set(val) receiver.wakeup - return end end - interrupt_process!(Proc.current) + Proc.check_interrupt! end def inspect + inspect_crit # doesn't entirely get rid of race conditions because # @block_set may still be mutated, but makes it less # likely i think? - dup.inspect_crit + # dup.inspect_crit end - protected - - def interrupt_process!(process) - process.interrupt!(Status[reason: Reason::Close.new(self)]) - end - - def inspect_crit + def repr dots = '*' * @block_set.size s = case @block_type when :none @@ -233,7 +278,21 @@ def inspect_crit o = @open ? 'o' : 'x' - "#" + "[#{@id} #{o} #{s}]" + end + + def to_s + repr + end + + protected + + def interrupt_process!(process, direction) + process.interrupt!(Status[reason: Reason::Close.new(self, direction)]) + end + + def inspect_crit + "#<#{self.class.name}#[#{repr}]>" end private diff --git a/lib/magritte/cli.rb b/lib/magritte/cli.rb index 46acc62..508fbae 100644 --- a/lib/magritte/cli.rb +++ b/lib/magritte/cli.rb @@ -23,11 +23,22 @@ def run_repl end def run_files - runner = Runner.new @files.each do |f| + runner = Runner.new status = runner.evaluate(f, File.read(f)) + + if runner.env.key?('__main__') + runner.evaluate(f, '__main__') + end + runner.synchronize { puts "% #{status.repr}" } if status.fail? end + + # XXX hack in case some threads don't exit + # will investigate this more when we have a + # proper vm + PRINTER.p(alive: (Thread.list - [Thread.current])) + exit! 0 end def parse_args diff --git a/lib/magritte/code.rb b/lib/magritte/code.rb index 9d41953..670bb5a 100644 --- a/lib/magritte/code.rb +++ b/lib/magritte/code.rb @@ -26,8 +26,10 @@ def s_(env=nil, &b) include DSL - def initialize(&block) + attr_reader :loc + def initialize(loc=nil, &block) @block = block + @loc ||= block.source_location end def run @@ -44,10 +46,6 @@ def spawn_collect(env = nil) def inspect "#" end - - def loc - @block.source_location.join(':') - end end class PlainCode < Code @@ -105,6 +103,9 @@ def p_(&block) end def as_code + trace = Proc.current? && Proc.current.trace.last + loc = trace ? trace.range.first : @block.source_location + Code.new(&@block) end diff --git a/lib/magritte/env.rb b/lib/magritte/env.rb index ee8622d..5a3bd7e 100644 --- a/lib/magritte/env.rb +++ b/lib/magritte/env.rb @@ -53,6 +53,14 @@ def output(n) @own_outputs[n] or parent :output, n end + def set_input(n, ch) + @own_inputs[n] = ch + end + + def set_output(n, ch) + @own_outputs[n] = ch + end + def each_input (0..32).each do |x| ch = input(x) or return @@ -75,6 +83,10 @@ def stdout output(0) end + def up + @parent || self + end + def splice(new_parent) Env.new(new_parent, @keys.dup, @own_inputs.dup, @own_outputs.dup) end diff --git a/lib/magritte/frame.rb b/lib/magritte/frame.rb index 4bb6d24..fd90468 100644 --- a/lib/magritte/frame.rb +++ b/lib/magritte/frame.rb @@ -1,20 +1,37 @@ module Magritte class Frame attr_reader :env, :lex_env + attr_reader :compensations def initialize(p, env, lex_env = nil) @env = env #@lex_env = lex_env || p.lex_env @proc = p @compensations = [] + @tail = false + @elim = false end + def tail!; @tail = true end + def tail?; @tail end + + def elim!; @compensations = []; @elim = true end + def elim?; @elim end + def compensate(status) - @compensations.each(&:run) + @tail = false + while c = @compensations.shift + c.run + end + unregister_channels end def checkpoint - @compensations.each(&:run_checkpoint) + @tail = false + while c = @compensations.shift + c.run_checkpoint + end + unregister_channels end @@ -23,11 +40,15 @@ def add_compensation(comp) end def repr - "f" + "f:[>#{@env.stdout} <#{@env.stdin}]" + end + + def to_s + repr end def inspect - "#" + "#" end def thread @@ -39,8 +60,8 @@ def open_channels @env.each_output { |c| c.add_writer(self) } end - private def unregister_channels + PRINTER.p unregister_channels: [@env.stdin.to_s, @env.stdout.to_s] @env.each_input { |c| c.remove_reader(self) } @env.each_output { |c| c.remove_writer(self) } end diff --git a/lib/magritte/free_vars.rb b/lib/magritte/free_vars.rb index 380b851..81821a0 100644 --- a/lib/magritte/free_vars.rb +++ b/lib/magritte/free_vars.rb @@ -4,6 +4,61 @@ def self.scan(node) Scanner.new.collect(node, Set.new) end + class Grouper < Tree::Visitor + def self.group(node) + new.visit(node) + end + + def visit_group(node) + elems = node.elems.map { |e| visit(e) } + + out = [] + while elems.any? + head = elems.shift + + @pre_decls = [] + @decls = [] + while head && declaration?(head) + @pre_decls << pre_decl(head) + @decls << decl_to_mut(head) + head = elems.shift + end + + if @decls.any? + out.concat(@pre_decls) + out.concat(@decls) + end + + out << head if head + end + + AST::Group[out] + end + + protected + def pre_decl(node) + AST::Assignment[node.lhs, [AST::String['__undef__']]] + end + + def decl_to_mut(node) + AST::Assignment[[AST::Variable[node.lhs[0].value]], node.rhs] + end + + def declaration?(node) + return false unless node.is_a? AST::Assignment + return false unless node.lhs.size == 1 && node.rhs.size == 1 + return false unless node.lhs[0].is_a? AST::String + return true if node.rhs[0].is_a? AST::Lambda + + # re-declaration of a constant resets the group + return false if @decls.any? { |d| d.lhs[0].name == node.lhs[0].value } + return true if node.rhs[0].is_a? AST::String + return true if node.rhs[0].is_a? AST::Number + + false + end + end + class BinderScanner < Tree::Collector def visit_binder(node) Set.new([node.name]) @@ -27,20 +82,29 @@ def visit_lambda(node, bound_vars) def visit_group(node, bound_vars) out = Set.new so_far = Set.new + node.elems.each do |elem| case elem when AST::Assignment recursive = so_far.dup + elem.lhs.each do |binder| recursive << binder.value if binder.is_a?(AST::String) out.merge(shadow(binder, bound_vars, so_far)) end elem.rhs.each do |el| - out.merge(shadow(el, bound_vars, el.is_a?(AST::Lambda) ? recursive : so_far)) + out.merge(shadow(el, bound_vars, so_far)) end so_far = recursive + # when AST::LiftGroup + # # pre-declare all assignments + # elem.elems.each { |assn| so_far << assn.lhs[0].value } + + # elem.elems.each do |assn| + # out.merge(shadow(assn, bound_vars, so_far)) + # end else out.merge(shadow(elem, bound_vars, so_far)) end @@ -49,6 +113,22 @@ def visit_group(node, bound_vars) out end + def visit_command(node, bound_vars) + out = Set.new + head, *rest = node.vec + if head.is_a?(AST::String) && bound_vars.include?(head.value) + out << head.value + else + out.merge(visit(head, bound_vars)) + end + + rest.each do |node| + out.merge(visit(node, bound_vars)) + end + + out + end + def shadow(node, bound_vars, shadow_vars) visit(node, bound_vars + shadow_vars) - shadow_vars end diff --git a/lib/magritte/interpret.rb b/lib/magritte/interpret.rb index 944914b..e30363f 100644 --- a/lib/magritte/interpret.rb +++ b/lib/magritte/interpret.rb @@ -9,6 +9,7 @@ class Interpreter < Tree::Walker include Code::DSL def initialize(root) + PRINTER.p(:interpret => root) @root = root @free_vars = FreeVars.scan(@root) end @@ -51,6 +52,7 @@ def visit_command(node) error!("Empty command") unless command + PRINTER.puts "> #{node.range}" command.call(args, node.range) end @@ -68,7 +70,19 @@ def visit_group(node) def visit_block(node) Proc.enter_frame(Proc.current.env.extend) do - visit_exec(node.group) + elems = node.group.elems.dup + last = elems.pop + + elems.each do |elem| + visit_exec(elem) + end + + if last + Proc.current.frame.tail! + visit_exec(last) + else + Status.normal + end end end @@ -87,7 +101,10 @@ def visit_pipe(node) end def visit_spawn(node) - s_ { visit_exec(node.expr) }.go + s_ { + Proc.current.frame.tail! + visit_exec(node.expr) + }.go Status.normal end @@ -130,11 +147,10 @@ def visit_access(node) end def visit_environment(node) - env = Proc.current.env.extend - Proc.enter_frame(env) do + env = Std.in_new_env(Proc.current.env) do visit_exec(node.body) end - env.unhinge! + yield Value::Environment.new(env) end @@ -143,6 +159,7 @@ def visit_with(node) outputs = [] redir_vals = node.redirects.map { |redirect| [redirect.direction, visit_collect(redirect.target).first] } redir_vals.each do |(dir, c)| + PRINTER.p("redirect #{dir} #{c.class} #{c.inspect}") error!("Cannot redirect #{dir} #{c.repr}") unless c.is_a?(Value::Channel) if dir == :< inputs << c.channel diff --git a/lib/magritte/lexer.rb b/lib/magritte/lexer.rb index d4ff9c4..0319669 100644 --- a/lib/magritte/lexer.rb +++ b/lib/magritte/lexer.rb @@ -27,15 +27,35 @@ class Token :lbrace => :rbrace, } CONTINUE = Set.new([ + :eof, :pipe, :write_to, :read_from, + :equal, :d_amp, :d_bar, :d_per, :d_bang, :d_per_bang, :arrow, + :rbrace, + :rbrack, + :rparen + ]) + + SKIP = Set.new([ + :lparen, + :lbrace, + :lbrack, + :arrow, + :equal, + :write_to, + :read_from, + :d_per, + :d_per_bang, + :d_amp, + :d_bar, + :pipe ]) FREE_NL = Set.new([ @@ -52,6 +72,10 @@ def continue? CONTINUE.include?(@type) end + def skip? + SKIP.include?(@type) + end + def nest? NESTED_PAIRS.key?(@type) end @@ -98,104 +122,117 @@ def initialize(source_name, string) @scanner = StringScanner.new(string) @line = 1 @col = 0 - skip_lines + + # p :lex => string + + skip_ws + advance while peek.is?(:nl) end attr_reader :source_name def peek - @peek ||= self.next + @peek ||= self.advance end def next + # p :next + tok = advance + + loop do + # puts "loop: #{tok.repr} #{peek.repr}" + if tok.is?(:nl) && peek.is?(:nl) + # puts "CONSOLIDATE" + advance + elsif tok.skip? && peek.is?(:nl) + advance while peek.is?(:nl) + # puts "SKIP: #{tok.repr}" + return tok + elsif tok.is?(:nl) && peek.continue? + out = peek + advance + # puts "CONTINUE: #{out.repr}" + return out + else + # puts "NORMAL: #{tok.repr}" + return tok + end + end + end + + def advance + out = advance_ + # puts "advance: #{out.repr} --- #{@match.inspect} --- #{@scanner.peek(5).inspect}" + out + end + + def advance_ if @peek p = @peek @peek = nil return p end - if @scanner.eos? - return token(:eof) - elsif scan /[\n;]|(#[^\n]*)/ - skip_lines - return token(:nl) - elsif scan /[(]/ - skip_lines - return token(:lparen) - elsif scan /[)]/ - skip_ws - return token(:rparen) - elsif scan /[{]/ - skip_lines - return token(:lbrace) - elsif scan /[}]/ - skip_ws - return token(:rbrace) - elsif scan /\[/ - skip_lines - return token(:lbrack) - elsif scan /\]/ - skip_ws - return token(:rbrack) - elsif scan /\=>/ - skip_ws - return token(:arrow) - elsif scan /[=]/ - skip_ws - return token(:equal) - elsif scan // - skip_ws - return token(:gt) - elsif scan /%%!/ - skip_ws - return token(:d_per_bang) - elsif scan /%%/ - skip_ws - return token(:d_per) - elsif scan /!!/ - skip_ws - return token(:d_bang) - elsif scan /&&/ - skip_ws - return token(:d_amp) - elsif scan /\|\|/ - skip_ws - return token(:d_bar) - elsif scan /&/ - skip_ws - return token(:amp) - elsif scan /\|/ - skip_ws - return token(:pipe) - elsif scan /!/ - skip_ws - return token(:bang) - elsif scan /[$](\w+)/ - skip_ws - return token(:var, group(1)) - elsif scan /[%](\w+)/ - skip_ws - return token(:lex_var, group(1)) - elsif scan /[?](\w+)/ - skip_ws - return token(:bind, group(1)) - elsif scan /([-]?[0-9]+([\.][0-9]*)?)/ - skip_ws - return token(:num, group(1)) - elsif scan /"((?:\\.|[^"])*)"/ - skip_ws - return token(:string, group(1)) - elsif scan /'(.*?)'/m - skip_ws - return token(:string, group(1)) - elsif scan /([_.\/a-zA-Z0-9-]+)/ + begin + if @scanner.eos? + return token(:eof) + elsif scan /[\n;]|(#[^\n]*)/ + return token(:nl) + elsif scan /[(]/ + return token(:lparen) + elsif scan /[)]/ + return token(:rparen) + elsif scan /[{]/ + return token(:lbrace) + elsif scan /[}]/ + return token(:rbrace) + elsif scan /\[/ + return token(:lbrack) + elsif scan /\]/ + return token(:rbrack) + elsif scan /\=>/ + return token(:arrow) + elsif scan /[=]/ + return token(:equal) + elsif scan // + return token(:gt) + elsif scan /%%!/ + return token(:d_per_bang) + elsif scan /%%/ + return token(:d_per) + elsif scan /!!/ + return token(:d_bang) + elsif scan /&&/ + return token(:d_amp) + elsif scan /\|\|/ + return token(:d_bar) + elsif scan /&/ + return token(:amp) + elsif scan /\|/ + return token(:pipe) + elsif scan /!/ + return token(:bang) + elsif scan /[$]([\w-]+)/ + return token(:var, group(1)) + elsif scan /[%]([\w-]+)/ + return token(:lex_var, group(1)) + elsif scan /[?](\w+)/ + return token(:bind, group(1)) + elsif scan /([-]?[0-9]+([\.][0-9]*)?)/ + return token(:num, group(1)) + elsif scan /"((?:\\.|[^"])*)"/ + return token(:string, group(1)) + elsif scan /'(.*?)'/m + return token(:string, group(1)) + elsif scan /([_.\/a-zA-Z0-9-]+)/ + return token(:bare, group(1)) + else + error!("Unknown token near #{@scanner.peek(3).inspect}") + end + ensure skip_ws - return token(:bare, group(1)) - else - error!("Unknown token near #{@scanner.peek(3)}") end end @@ -257,6 +294,10 @@ def initialize(first, last) def repr "#{first.source_name}@#{first.repr}~#{last.repr}" end + + def to_s + repr + end end # This function is called when we have instantiated a lexer @@ -278,7 +319,7 @@ def scan(re) prev_pos = current_pos if @scanner.scan(re) @match = @scanner.matched - @groups = @scanner.captures + @groups = [@scanner[1], @scanner[2], @scanner[3]] # XXX HACK XXX update_line_col(@match) @last_range = Range.new(prev_pos, current_pos) true @@ -322,7 +363,16 @@ def skip_ws end def skip_lines - skip /((#[^\n]*)?\n[ \t;]*)*[ \t;]*/m + skip %r{ + \s* + ( + ([#][^\n]*\n?)\s* + | + [\n;]\s* + )* + \s* + }mx + p :skip_lines => [@scanner.matched, @scanner.peek(5)] end def skip(re) diff --git a/lib/magritte/log.rb b/lib/magritte/log.rb index f19189b..8b33ad6 100644 --- a/lib/magritte/log.rb +++ b/lib/magritte/log.rb @@ -19,6 +19,11 @@ def p(*); end end class LogPrinter + def self.thread_name(t) + t.inspect =~ /0x\h+/ + $& + end + def initialize(prefix) @prefix = prefix end @@ -28,8 +33,7 @@ def current_log end def fname - Thread.current.inspect =~ /0x\h+/ - $& + LogPrinter.thread_name(Thread.current) end def with_file(&b) diff --git a/lib/magritte/parser.rb b/lib/magritte/parser.rb index 8fb240e..31499d4 100644 --- a/lib/magritte/parser.rb +++ b/lib/magritte/parser.rb @@ -22,7 +22,7 @@ def error!(skel, msg) end def parse(skel) - parse_root(skel) + FreeVars::Grouper.group(parse_root(skel)) end def parse_root(skel) @@ -34,11 +34,7 @@ def parse_root(skel) def parse_line(item) item.match(starts(token(:amp), ~_)) do |body| - return AST::Spawn[parse_line(body)] - end - - item.match(rsplit(~_, token(:pipe), ~_)) do |before, after| - return AST::Pipe[parse_line(before), parse_command(after)] + return AST::Spawn[AST::Block[AST::Group[[parse_line(body)]]]] end # Match any line that has an equal sign @@ -46,6 +42,10 @@ def parse_line(item) return parse_assignment(lhs, rhs) end + item.match(rsplit(~_, token(:pipe), ~_)) do |before, after| + return AST::Pipe[parse_line(before), parse_command(after)] + end + # Parse double & item.match(lsplit(~_, token(:d_amp), ~_)) do |lhs, rhs| # Check for double ! @@ -127,11 +127,21 @@ def parse_command(command) next if command.match(starts(~any(token(:gt), token(:lt)), ~_)) do |dir, rest| error!(command, 'redirect at end') if rest.elems.empty? target, *rest = rest.elems - direction = dir.token?(:gt) ? :> : :< + if target.nested?(:lparen) error!(target, 'TODO: redir to paren') + # also consider what happens to + # foo > (bar baz)!zot + # ... probably just an error, if parens have special semantics else - redirects << AST::Redirect[direction, parse_term(target)] + targets = [target] + targets.concat(rest.shift(2)) while rest.first && rest.first.match(token(:bang)) + targets = Skeleton::Item[targets] + direction = dir.token?(:gt) ? :> : :< + parsed_targets = parse_terms(targets) + + error!(targets, 'invalid redirect') if parsed_targets.size != 1 + redirects << AST::Redirect[direction, parsed_targets.first] end command = Skeleton::Item[rest] diff --git a/lib/magritte/proc.rb b/lib/magritte/proc.rb index de2886d..ed00c2b 100644 --- a/lib/magritte/proc.rb +++ b/lib/magritte/proc.rb @@ -6,6 +6,14 @@ class Interrupt < RuntimeError def initialize(status) @status = status end + + def to_s + "!interrupt[#{@status.repr}]" + end + + def inspect + "#<#{self.class.name} #{@status.repr}>" + end end def self.current @@ -24,6 +32,10 @@ def self.enter_frame(*args, &b) current.send(:enter_frame, *args, &b) end + def self.check_interrupt! + current.check_interrupt! + end + def self.spawn(code, env) start_mutex = Mutex.new start_mutex.lock @@ -36,6 +48,7 @@ def self.spawn(code, env) Thread.current[:status] = begin code.run rescue Interrupt => e + PRINTER.p("root compensation #{Proc.current.inspect}") Proc.current.compensate(e) e.status end @@ -48,9 +61,10 @@ def self.spawn(code, env) Thread.current[:status] = status raise ensure + PRINTER.puts("shutting down, final ensure #{Proc.current.inspect}") @alive = false Proc.current.checkpoint_all - PRINTER.puts('exiting') + PRINTER.puts("exiting #{Proc.current.inspect}") end end @@ -66,7 +80,7 @@ def self.spawn(code, env) end def inspect - "#" + "#" end def wait @@ -94,8 +108,9 @@ def initialize(thread, code, env) @code = code @env = env @stack = [] + @interrupts = [] - @interrupt_mutex = LogMutex.new :interrupt + @mutex = LogMutex.new "interrupt_#{LogPrinter.thread_name(@thread)}" end def env @@ -103,9 +118,14 @@ def env end def frame + binding.pry if @stack.empty? @stack.last end + def own_thread! + raise Exception.new('cannot call on different thread') unless @thread == Thread.current + end + def start @alive = true @stack << Frame.new(self, @env) @@ -119,16 +139,31 @@ def alive? end def interrupt!(status) - @interrupt_mutex.synchronize do - return unless alive? + @mutex.synchronize do + next unless alive? - # will run cleanup in the thread via the ensure block - @thread.raise(Interrupt.new(status)) + @interrupts << Interrupt.new(status) + @thread.run end end + def check_interrupt! + own_thread! + + ex = @mutex.synchronize do + PRINTER.puts("check_interrupt! #{@interrupts.inspect}") + @mutex.log("check_interrupt! #{@interrupts.inspect}") + @interrupts.shift + end + + raise ex if ex + end + def crash!(msg=nil) + own_thread! + interrupt!(Status[:crash, reason: Reason::Crash.new(msg)]) + check_interrupt! end def sleep @@ -148,25 +183,42 @@ def stdin end def add_compensation(comp) + own_thread! + frame.add_compensation(comp) end def compensate(status) + own_thread! + # if the stack is empty there is nothing to do + + PRINTER.p("compensate popping #{@stack.size}") + frame = @stack.pop - frame.compensate(status) + frame && frame.compensate(status) + check_interrupt! end def checkpoint - frame = @stack.pop - frame.checkpoint + own_thread! + + PRINTER.p("checkpoint popping #{@stack.size}") + frame = @stack.last + frame && frame.checkpoint + @stack.pop + check_interrupt! end def compensate_all(e) - compensate(e) until @stack.empty? + own_thread! + + (compensate(e) rescue Interrupt) until @stack.empty? end def checkpoint_all - checkpoint until @stack.empty? + own_thread! + + (checkpoint rescue Interrupt) until @stack.empty? end class Tracepoint @@ -180,6 +232,9 @@ def initialize(callable, range) end def with_trace(callable, range, &b) + own_thread! + + PRINTER.p("trace: #{callable.name} #{range}") @trace << Tracepoint.new(callable, range) yield ensure @@ -189,25 +244,59 @@ def with_trace(callable, range, &b) attr_reader :trace protected + def re_raise?(e) + return true if @stack.empty? + + reason = e.status.reason + return true unless reason.is_a?(Reason::Close) + + test = proc { |c| return true if c == reason.channel } + + case reason.direction + when :< then env.each_input(&test) + when :> then env.each_output(&test) + end + + false + end + def enter_frame(*args, &b) + own_thread! + stack_size = @stack.size - @interrupt_mutex.synchronize do - frame = Frame.new(self, *args) - PRINTER.p :stack => @stack - @stack << frame + frame = Frame.new(self, *args) + frame.open_channels + + if @stack.last.tail? + PRINTER.p("tail-popping #{@stack.size} #{@stack.last.inspect}") + tail = @stack.pop + tail.unregister_channels + frame.compensations.concat(tail.compensations) + tail.elim! end - frame.open_channels + PRINTER.p :stack => @stack + @stack << frame + + + PRINTER.p 'channels opened' out = yield rescue Interrupt => e - compensate(e) - PRINTER.p :interrupt => @stack - raise + compensate(e) unless frame.elim? + PRINTER.p :interrupt => [e.status, @stack] + if re_raise?(e) + raise + else + e.status + end else - checkpoint - PRINTER.p :checkpoint => @stack + unless frame.elim? + checkpoint + PRINTER.p :checkpoint => @stack + end + out end end diff --git a/lib/magritte/runner.rb b/lib/magritte/runner.rb index a617999..e473856 100644 --- a/lib/magritte/runner.rb +++ b/lib/magritte/runner.rb @@ -1,12 +1,22 @@ module Magritte class Runner - attr_reader :input, :output + attr_reader :input, :output, :env def initialize @mutex = Mutex.new - @output = Channel.new @input_num = 1 - @input = InputStreamer.new do + @env = Env.empty + setup_channels + Builtins.load(@env) + end + + def reset_channels + @env.stdin.reset! + @env.stdout.reset! + end + + def setup_channels + input = InputStreamer.new do begin source = @mutex.synchronize { @input_num += 1 @@ -25,7 +35,12 @@ def initialize end end - @env = Env.base.extend([@input], [@output]) + output = Streamer.new do |val| + @mutex.synchronize { puts val } + end + + @env.set_output(0, output) + @env.set_input(0, input) end def input_name @@ -42,19 +57,18 @@ def evaluate(source_name, source) ast = Parser.parse(Skeleton.parse(Lexer.new(source_name, source))) p = Proc.spawn(Code.new { Interpret.interpret(ast) }, @env) - o = Proc.spawn(Code.new { loop { puts @output.read.repr } }, Env.empty).start + status = p.wait - o.join rescue CompileError => e Status[:fail, reason: Reason::Compile.new(e)] - rescue Exception => e + rescue ::Interrupt => e Status[:fail, reason: Reason::UserInterrupt.new(e)] + rescue Exception => e + Status[:fail, reason: Reason::Bug.new(e)] else status ensure - # p && p.crash!("ended") - @output.reset! - @input.reset! + reset_channels end end end diff --git a/lib/magritte/skeleton.rb b/lib/magritte/skeleton.rb index caaa817..c8b7c17 100644 --- a/lib/magritte/skeleton.rb +++ b/lib/magritte/skeleton.rb @@ -151,7 +151,6 @@ def parse(lexer) # it can happen that we try to call free_nl? # on a nil object.... next if !self.open.nil? && self.open.free_nl? - next if (last && last.continue?) || lexer.peek.continue? @items << [] else y Token[token] diff --git a/lib/magritte/status.rb b/lib/magritte/status.rb index 1eace72..54b8b3d 100644 --- a/lib/magritte/status.rb +++ b/lib/magritte/status.rb @@ -18,9 +18,10 @@ def to_s end class Close < Base - attr_reader :channel - def initialize(channel) + attr_reader :channel, :direction + def initialize(channel, direction) @channel = channel + @direction = direction end def to_s diff --git a/lib/magritte/std.rb b/lib/magritte/std.rb index 9bf9155..b187782 100644 --- a/lib/magritte/std.rb +++ b/lib/magritte/std.rb @@ -1,10 +1,19 @@ module Magritte module Std extend self + require 'timeout' + require 'pry' def put(val) Proc.current.stdout.write(val) end + def in_new_env(env, &b) + new_env = env.extend + Proc.enter_frame(new_env, &b) + new_env.unhinge! + new_env + end + def get out = Proc.current.stdin.read # PRINTER.p get: [out, stdin] @@ -17,10 +26,20 @@ def call(h, a, range=nil) end def loop_channel(c, &b) - loop { b.call } + loop do + PRINTER.p :loop_channel => c + b.call + Proc.check_interrupt! + end rescue Proc::Interrupt => e reason = e.status.reason - raise unless reason && reason.is_a?(Reason::Close) && reason.channel == c + + if reason && reason.is_a?(Reason::Close) && reason.channel == c + PRINTER.puts("loop_channel rescued: #{e.status.repr}") + else + PRINTER.puts("loop_channel bypassed: #{e.status.repr}") + raise + end end def produce(&b) diff --git a/lib/magritte/streamer.rb b/lib/magritte/streamer.rb index f25f983..3bf1210 100644 --- a/lib/magritte/streamer.rb +++ b/lib/magritte/streamer.rb @@ -2,13 +2,17 @@ module Magritte class Streamer < Channel def initialize(&b) + setup_id @output = b - @mutex = Mutex.new @writers = Set.new @close_waiters = [] @open = true end + def to_s + "streamer##{@id}:#{@output.source_location.join(':')}" + end + def add_reader(*); end def remove_reader(*); end @@ -20,10 +24,9 @@ def remove_writer(p) next :nop unless @writers.empty? @open = false - :close + @close_waiters.each { |t| t.run } # if action == :close + @close_waiters.clear end - - @close_waiters.each { |t| t.run } if action == :close end def read @@ -34,18 +37,26 @@ def write(val) @output.call(val) end + require 'pry' def wait_for_close + PRINTER.p("waiting for close #{self} #{@writers}") + @mutex.synchronize do next unless @open @close_waiters << Thread.current @mutex.sleep end + + Proc.current? && Proc.check_interrupt! end def reset! @mutex.synchronize do - @open = true + @mutex.log 'reset' + @close_waiters.each { |t| t.run } # if action == :close @close_waiters.clear + + @open = true @writers.clear end end @@ -57,6 +68,7 @@ def inspect_crit class InputStreamer < Channel def initialize(&b) + setup_id @block = b @readers = Set.new @mutex = Mutex.new @@ -64,6 +76,10 @@ def initialize(&b) @queue = [] end + def to_s + "input-streamer##{@id}:#{@block.source_location.join(':')}" + end + def add_reader(*); end def add_writer(*); end def remove_reader(*); end @@ -80,7 +96,7 @@ def read return @queue.shift end - interrupt_process!(Proc.current) + interrupt_process!(Proc.current, :<) end def reset! @@ -107,12 +123,17 @@ class Collector < Streamer attr_reader :collection def initialize + setup_id @collection = [] super { |val| @mutex.synchronize { @collection << val } } end + def to_s + "collector##{@id}:[#{@collection.size}]" + end + def inspect - "#" + "#" end end end diff --git a/lib/magritte/tree.rb b/lib/magritte/tree.rb index 2868533..53dc87e 100644 --- a/lib/magritte/tree.rb +++ b/lib/magritte/tree.rb @@ -166,7 +166,15 @@ def ==(o) alias eql? == def inspect - "#<#{self.class}#{attrs.inspect}>" + "#<#{self.class}#{parrepr}>" + end + + def repr + "#{attrs.inspect}" + end + + def parrepr + "(#{repr})" end def hash diff --git a/lib/magritte/value.rb b/lib/magritte/value.rb index 7a84f0a..2b494c6 100644 --- a/lib/magritte/value.rb +++ b/lib/magritte/value.rb @@ -238,6 +238,10 @@ def typename 'compensation' end + def repr + "" + end + def run @action.call([], @range) end diff --git a/mag/prelude.mag b/mag/prelude.mag index 88680d4..b551229 100644 --- a/mag/prelude.mag +++ b/mag/prelude.mag @@ -2,6 +2,9 @@ exec = [] +(incr ?x) = add 1 $x +(decr ?x) = sub 1 $x + (times ?n ?fn (?args)) = ( in = (stdin) range %n | each (_ => exec %fn (for %args) < %in) @@ -19,6 +22,9 @@ exec = [] (prob ?total ?amt) = (lt %amt (mul %total (rand))) +(rand-between ?l ?h) = (add $l (round 0 (mul (sub $l $h) (rand)))) +(is-odd ?x) = (eq 1 (mod 2 $x)) +(is-even ?x) = (eq 0 (mod 2 $x)) (sample) = ( hold = (get) @@ -46,7 +52,7 @@ exec = [] (take-until ?pred) = (each (?e => %pred %e && put %e !! false)) LOG = (stdout) -(log ?msg) = (put ["log:" %msg] > $LOG) +(log (?msg)) = (put ["log:" (for %msg)] > $LOG) null = (make-null) (through ?o) = ( diff --git a/spec/code_spec.rb b/spec/code_spec.rb index ac21f62..1b69661 100644 --- a/spec/code_spec.rb +++ b/spec/code_spec.rb @@ -1,26 +1,5 @@ describe Magritte::Code do - def with_no_dangling_threads(&b) - orig_threads = Thread.list - out = yield - - begin - dangling = Thread.list - orig_threads - - assert { dangling.empty? } - out - rescue Minitest::Assertion - retry_count ||= 0 - retry_count += 1 - raise if retry_count > 20 - - # yield the current thread to allow other threads to be cleaned up - # since sometimes it takes a bit of time for thread.raise to actually - # kill the thread and there's no way to wait for it :\ - sleep 0.1 - - retry - end - end + include DanglingThreads let(:output) { with_no_dangling_threads { code.spawn_collect } } @@ -87,11 +66,11 @@ def self.code(&b) s { for_ (0..3) } .p { loop { put (get * 2) } }.call - raise "never reached" + put 20 end it 'interrupts the parent process' do - assert { output == [0, 2, 4, 6] } + assert { output == [0, 2, 4, 6, 20] } end end diff --git a/spec/free_vars_spec.rb b/spec/free_vars_spec.rb index 5342b79..3c26780 100644 --- a/spec/free_vars_spec.rb +++ b/spec/free_vars_spec.rb @@ -2,7 +2,7 @@ abstract(:input) let(:lex) { Magritte::Lexer.new("test",input) } let(:skel) { Magritte::Skeleton::Parser.parse(lex) } - let(:ast) { Magritte::Parser.parse_root(skel) } + let(:ast) { Magritte::Parser.parse(skel) } let(:result) { Magritte::FreeVars.scan(ast) } def free_vars(node = nil) diff --git a/spec/interpret/basic_spec.rb b/spec/interpret/basic_spec.rb index 302e4d1..dd73145 100644 --- a/spec/interpret/basic_spec.rb +++ b/spec/interpret/basic_spec.rb @@ -23,7 +23,7 @@ result "1" end - interpret "early exist for collectors" do + interpret "early exit for collectors" do source <<-EOF put 1 2 3 4 5 6 7 8 9 10 | (& drain; & drain) EOF diff --git a/spec/interpret/env_spec.rb b/spec/interpret/env_spec.rb index 0d86aba..d58a511 100644 --- a/spec/interpret/env_spec.rb +++ b/spec/interpret/env_spec.rb @@ -47,4 +47,14 @@ results %w(2 1) end + + interpret "redirecting" do + source <<-EOF + e = { x = (make-channel) } + & put 3 > $e!x + get < $e!x + EOF + + results %w(3) + end end diff --git a/spec/interpret/interrupt_spec.rb b/spec/interpret/interrupt_spec.rb index 263d2e2..81fc9be 100644 --- a/spec/interpret/interrupt_spec.rb +++ b/spec/interpret/interrupt_spec.rb @@ -13,10 +13,10 @@ interpret "interrupts" do source <<-EOF c = (make-channel) - exec (=> ( + exec (=> put 1 %% (put comp > %c) put 2 3 4 5 6 - )) | take 2 + ) | take 2 get < $c EOF @@ -29,11 +29,13 @@ & put 1 2 3 > $c drain < $c (dr) = (put (get); dr) - dr < $c - # we never get here, because we get - # interrupted by the above - put 4 + ( + dr + # we never get here, because we get + # interrupted by the above + put 4 + ) < $c EOF results %w(1 2 3) @@ -46,11 +48,27 @@ get < $c get < $c - # we never get here, because we get interrupted - # by the above + # we continue here, because we don't have $c as an input put 4 EOF - results %w(1) + results %w(1 4) + end + + interpret "the bug" do + source <<-EOF + c = (make-channel) + (f) = ( + & put 1 > $c + put 2 + ) + + x = (f) + + get < $c + put $x + EOF + + results %w(1 2) end end diff --git a/spec/interpret/lambda_spec.rb b/spec/interpret/lambda_spec.rb index 5f6e2df..d519ab3 100644 --- a/spec/interpret/lambda_spec.rb +++ b/spec/interpret/lambda_spec.rb @@ -171,7 +171,7 @@ results ["1", "hello", "[2 3]"] end - interpret 'thing' do + interpret 'assigned variables within lambdas' do source <<-EOF range 3 | each (?x => r = %x @@ -181,4 +181,30 @@ results %w(0 1 2) end + + interpret 'isolated closures' do + source <<-EOF + e = { + (f1) = (drain) + (f2) = (put 2 | f1) + } + + $e!f2 + EOF + + results %w(2) + end + + interpret 'lifting' do + source <<-EOF + e = { + (f) = (%g 1; %g 2) + (g ?x) = (inc %x) + } + + $e!f + EOF + + results %w(2 3) + end end diff --git a/spec/lexer_spec.rb b/spec/lexer_spec.rb index 040bfbb..e49b13d 100644 --- a/spec/lexer_spec.rb +++ b/spec/lexer_spec.rb @@ -17,7 +17,7 @@ } it "parses basic delimiters" do - assert { tokens == [_token(:lparen), _token(:lbrack), _token(:var,"hoge"), _token(:nl), _token(:eof)] } + assert { tokens == [_token(:lparen), _token(:lbrack), _token(:var,"hoge"), _token(:eof)] } end end @@ -29,7 +29,7 @@ } it "parses basic keywords" do - assert { tokens == [_token(:lbrack), _token(:lparen), _token(:lbrace), _token(:rbrace), _token(:rparen), _token(:rbrack), _token(:equal), _token(:lex_var, "a"), _token(:nl), _token(:eof)] } + assert { tokens == [_token(:lbrack), _token(:lparen), _token(:lbrace), _token(:rbrace), _token(:rparen), _token(:rbrack), _token(:equal), _token(:lex_var, "a"), _token(:eof)] } end end @@ -41,7 +41,7 @@ } it "parses basic function header" do - assert { tokens == [_token(:lparen), _token(:bare, "func"), _token(:bind, "n"), _token(:rparen), _token(:nl), _token(:eof)] } + assert { tokens == [_token(:lparen), _token(:bare, "func"), _token(:bind, "n"), _token(:rparen), _token(:eof)] } end end @@ -53,7 +53,7 @@ } it "parses tokens correctly" do - assert { tokens == [_token(:d_bar), _token(:d_amp), _token(:d_bang), _token(:d_per), _token(:d_per_bang), _token(:nl), _token(:eof)] } + assert { tokens == [_token(:d_bar), _token(:d_amp), _token(:d_bang), _token(:d_per), _token(:d_per_bang), _token(:eof)] } end end @@ -65,7 +65,7 @@ } it "parses operators" do - assert { tokens == [_token(:pipe), _token(:amp), _token(:equal), _token(:equal), _token(:arrow), _token(:nl), _token(:eof)] } + assert { tokens == [_token(:pipe), _token(:amp), _token(:equal), _token(:equal), _token(:arrow), _token(:eof)] } end end @@ -77,7 +77,7 @@ } it "parses numbers correctly" do - assert { tokens == [_token(:num, "2"), _token(:num, "6.28"), _token(:num, "0.00001"), _token(:num, "1."), _token(:num,"-5.4"), _token(:nl), _token(:eof)] } + assert { tokens == [_token(:num, "2"), _token(:num, "6.28"), _token(:num, "0.00001"), _token(:num, "1."), _token(:num,"-5.4"), _token(:eof)] } end end @@ -89,7 +89,7 @@ } it "parses strings correctly" do - assert { tokens == [_token(:string, "asksnz-zwjdfqw345 r8 ewn ih2wu\\\" wihf002+4-r9+***.m.-< \\\""), _token(:nl), _token(:eof)] } + assert { tokens == [_token(:string, "asksnz-zwjdfqw345 r8 ewn ih2wu\\\" wihf002+4-r9+***.m.-< \\\""), _token(:eof)] } end end diff --git a/spec/parser_spec.rb b/spec/parser_spec.rb deleted file mode 100644 index 9a5b42d..0000000 --- a/spec/parser_spec.rb +++ /dev/null @@ -1,158 +0,0 @@ -describe Magritte::Parser do - abstract(:input) - - let(:lex) { Magritte::Lexer.new("test",input) } - let(:skel) { Magritte::Skeleton::Parser.parse(lex) } - let(:ast) { Magritte::Parser.parse_root(skel) } - - describe "a vector" do - let(:input) { - """ - put [a b c] - """ - } - - it "parses correctly" do - assert { ast.elems.size == 1 } - assert { ast.elems.first.is_a?(Magritte::AST::Command) } - assert { ast.elems.first.vec.size == 2 } - assert { ast.elems.first.vec.inspect == "[#, #, #, #]]>]" } - end - end - - describe "access" do - let(:input) { - """ - put $foo!bar - """ - } - - it "parses correctly" do - assert { ast.elems.size == 1 } - assert { ast.elems.first.vec[1].is_a?(Magritte::AST::Access) } - end - end - - describe "pipe" do - let(:input) { - """ - f $a | put - """ - } - - it "parses correctly" do - assert { ast.elems.size == 1 } - assert { ast.elems.first.is_a?(Magritte::AST::Pipe) } - assert { ast.elems.first.producer.is_a?(Magritte::AST::Command) } - assert { ast.elems.first.consumer.is_a?(Magritte::AST::Command) } - end - end - - describe "spawn" do - let(:input) { - """ - & command arg1 - """ - } - - it "parses correctly" do - assert { ast.elems.size == 1 } - assert { ast.elems.first.is_a?(Magritte::AST::Spawn) } - assert { ast.elems.first.expr.is_a?(Magritte::AST::Command) } - end - end - - describe "rescue operators" do - let(:input) { - """ - a && b || c !! d - """ - } - - it do - assert { ast.elems.size == 1 } - assert { ast.elems.first.is_a?(Magritte::AST::And) } - assert { ast.elems.first.lhs.is_a?(Magritte::AST::Command) } - assert { ast.elems.first.lhs.vec.size == 1 } - assert { ast.elems.first.lhs.vec.first.value == "a" } - assert { ast.elems.first.rhs.is_a?(Magritte::AST::Else) } - assert { ast.elems.first.rhs.lhs.is_a?(Magritte::AST::Or) } - end - end - - describe "switch statement" do - let(:input) { - """ - cond && c !! cond2 && c2 !! cond3 && c3 !! c4 - """ - } - - it do - assert { ast.elems.size == 1 } - assert { ast.elems.first.is_a?(Magritte::AST::Else) } - assert { ast.elems.first.rhs.is_a?(Magritte::AST::Else) } - assert { ast.elems.first.rhs.rhs.is_a?(Magritte::AST::Else) } - assert { ast.elems.first.lhs.is_a?(Magritte::AST::And) } - assert { ast.elems.first.rhs.lhs.is_a?(Magritte::AST::And) } - assert { ast.elems.first.rhs.rhs.lhs.is_a?(Magritte::AST::And) } - assert { ast.elems.first.rhs.rhs.rhs.vec.first.value == "c4" } - end - end - - describe "compensation" do - let(:input) { - """ - c a1 %% c2 a2 - """ - } - - it do - assert { ast.elems.size == 1 } - assert { ast.elems.first.is_a?(Magritte::AST::Compensation) } - assert { ast.elems.first.expr.is_a?(Magritte::AST::Command) } - assert { ast.elems.first.compensation.is_a?(Magritte::AST::Command) } - assert { ast.elems.first.expr.vec.size == 2 } - assert { ast.elems.first.expr.vec[0].value == "c" } - assert { ast.elems.first.expr.vec[1].value == "a1" } - assert { ast.elems.first.compensation.vec.size == 2 } - assert { ast.elems.first.compensation.vec[0].value == "c2" } - assert { ast.elems.first.compensation.vec[1].value == "a2" } - assert { ast.elems.first.unconditional == :conditional } - end - end - - describe "compensation with checkpoints" do - let(:input) { - """ - c a1 %%! c2 a2 - """ - } - - it do - assert { ast.elems.size == 1 } - assert { ast.elems.first.is_a?(Magritte::AST::Compensation) } - assert { ast.elems.first.expr.is_a?(Magritte::AST::Command) } - assert { ast.elems.first.compensation.is_a?(Magritte::AST::Command) } - assert { ast.elems.first.expr.vec.size == 2 } - assert { ast.elems.first.expr.vec[0].value == "c" } - assert { ast.elems.first.expr.vec[1].value == "a1" } - assert { ast.elems.first.compensation.vec.size == 2 } - assert { ast.elems.first.compensation.vec[0].value == "c2" } - assert { ast.elems.first.compensation.vec[1].value == "a2" } - assert { ast.elems.first.unconditional == :unconditional } - end - end - - describe "only one compensation per line" do - let(:input) { - """ - c a1 %% c2 a2 %% c3 - """ - } - - it do - err = assert_raises { ast } - assert { err.message =~ /\Aunrecognized syntax at test@/ } - end - end -end diff --git a/spec/support/dangling_threads.rb b/spec/support/dangling_threads.rb new file mode 100644 index 0000000..90fb8a0 --- /dev/null +++ b/spec/support/dangling_threads.rb @@ -0,0 +1,24 @@ +module DanglingThreads + def with_no_dangling_threads(&b) + orig_threads = Thread.list + out = yield + + begin + dangling = Thread.list - orig_threads + + assert { dangling.empty? } + out + rescue Minitest::Assertion + retry_count ||= 0 + retry_count += 1 + raise if retry_count > 20 + + # yield the current thread to allow other threads to be cleaned up + # since sometimes it takes a bit of time for thread.raise to actually + # kill the thread and there's no way to wait for it :\ + sleep 0.1 + + retry + end + end +end diff --git a/spec/support/interpret_helpers.rb b/spec/support/interpret_helpers.rb index ad32770..7bc8ed6 100644 --- a/spec/support/interpret_helpers.rb +++ b/spec/support/interpret_helpers.rb @@ -1,5 +1,9 @@ require "ostruct" +require_relative 'dangling_threads' + module InterpretHelpers + include DanglingThreads + def self.included(base) base.send(:extend, ClassMethods) @@ -8,12 +12,14 @@ def self.included(base) let(:lex) { Magritte::Lexer.new(input_name, input) } let(:skel) { Magritte::Skeleton::Parser.parse(lex) } - let(:ast) { Magritte::Parser.parse_root(skel) } + let(:ast) { Magritte::Parser.parse(skel) } let(:env) { Magritte::Builtins.load(Magritte::Env.empty) } let(:results) { ast; - collection, @status = Magritte::Spawn.s_ env do - Magritte::Interpret.interpret(ast) - end.collect_with_status + collection, @status = with_no_dangling_threads do + Magritte::Spawn.s_ env do + Magritte::Interpret.interpret(ast) + end.collect_with_status + end collection.map(&:to_s) } @@ -74,9 +80,9 @@ def results_size(len) end def debug - @result_expectations << proc do + @status_expectations.unshift(proc do binding.pry - end + end) end def evaluate(&b)