Add documentation support for lib, fun, union, cstruct, external, and type (RFC #11) (#15447)

Co-authored-by: Johannes Müller <[email protected]>

Author: nobodywasishere

spec/compiler/crystal/tools/doc/directives_spec.cr

@@ -47,6 +47,21 @@ describe Crystal::Doc::Generator do
       a_def.visibility.should eq("private")
     end
 
+    it "shows documentation for nested objects if a lib is marked with :showdoc:" do
+      program = top_level_semantic(<<-CRYSTAL, wants_doc: true).program
+        # :showdoc:
+        lib Foo
+          # docs for `foo`
+          fun foo
+        end
+        CRYSTAL
+
+      generator = Doc::Generator.new program, [""]
+
+      generator.must_include?(program.types["Foo"]).should be_true
+      generator.type(program.types["Foo"]).lookup_method("foo").should_not be_nil
+    end
+
     it "does not include documentation for methods within a :nodoc: namespace" do
       program = top_level_semantic(<<-CRYSTAL, wants_doc: true).program
         # :nodoc:
@@ -91,6 +106,19 @@ describe Crystal::Doc::Generator do
       generator.must_include?(generator.type(program.types["Foo"]).lookup_path("Baz")).should be_false
     end
 
+    it "does not include documentation for a :showdoc: fun inside a lib not marked with :showdoc:" do
+      program = top_level_semantic(<<-CRYSTAL, wants_doc: true).program
+        lib Foo
+          # :showdoc:
+          fun foo
+        end
+        CRYSTAL
+
+      generator = Doc::Generator.new program, [""]
+
+      generator.must_include?(program.types["Foo"]).should be_false
+    end
+
     it "doesn't show a method marked :nodoc: within a :showdoc: namespace" do
       program = top_level_semantic(<<-CRYSTAL, wants_doc: true).program
         # :showdoc:
@@ -105,5 +133,22 @@ describe Crystal::Doc::Generator do
       generator = Doc::Generator.new program, [""]
       generator.type(program.types["Foo"]).lookup_method("foo").should be_nil
     end
+
+    it "doesn't show a fun marked :nodoc: within a :showdoc: lib" do
+      program = top_level_semantic(<<-CRYSTAL, wants_doc: true).program
+        # :showdoc:
+        lib Foo
+          # :nodoc:
+          # Some docs for `foo`
+          fun foo
+
+          fun bar
+        end
+        CRYSTAL
+
+      generator = Doc::Generator.new program, [""]
+      generator.type(program.types["Foo"]).lookup_method("foo").should be_nil
+      generator.type(program.types["Foo"]).lookup_method("bar").should_not be_nil
+    end
   end
 end

spec/compiler/parser/parser_doc_spec.cr

@@ -16,6 +16,7 @@ describe "Parser doc" do
     {"alias", "alias Foo = Bar"},
     {"annotation", "@[Some]"},
     {"private def", "private def foo\nend"},
+    {"lib def", "lib MyLib\nend"},
   ].each do |(desc, code)|
     it "includes doc for #{desc}" do
       parser = Parser.new(%(
@@ -29,6 +30,53 @@ describe "Parser doc" do
     end
   end
 
+  [
+    {"type def", "type Foo = Bar"},
+    {"cstruct def", "struct Name\nend"},
+    {"union def", "union Name\nend"},
+    {"fun def", "fun name = Name"},
+    {"external var", "$errno : Int32"},
+  ].each do |(desc, code)|
+    it "includes doc for #{desc} inside lib def" do
+      parser = Parser.new(%(
+        lib MyLib
+          # This is Foo.
+          # Use it well.
+          #{code}
+        end
+        ))
+      parser.wants_doc = true
+      node = parser.parse
+      node.as(Crystal::LibDef).body.doc.should eq("This is Foo.\nUse it well.")
+    end
+  end
+
+  it "includes doc for cstruct fields" do
+    parser = Parser.new(<<-CRYSTAL)
+      lib MyLib
+        struct IntOrFloat
+          # This is Foo.
+          # Use it well.
+          some_int : Int32
+          # This is Foo.
+          # Use it well.
+          some_float, other_float : Float64
+        end
+      end
+      CRYSTAL
+
+    parser.wants_doc = true
+    node = parser.parse
+    node.as(Crystal::LibDef)
+      .body.as(Crystal::CStructOrUnionDef)
+      .body.as(Crystal::Expressions)
+      .expressions.each do |exp|
+      exp.as(Crystal::TypeDeclaration)
+        .var.as(Crystal::Var)
+        .doc.should eq("This is Foo.\nUse it well.")
+    end
+  end
+
   it "disables doc parsing inside defs" do
     parser = Parser.new(%(
       # doc 1

src/compiler/crystal/semantic/ast.cr

@@ -686,6 +686,7 @@ module Crystal
   end
 
   class External < Def
+    property? external_var : Bool = false
     property real_name : String
     property! fun_def : FunDef
     property call_convention : LLVM::CallConvention?

src/compiler/crystal/semantic/top_level_visitor.cr

@@ -556,6 +556,8 @@ class Crystal::TopLevelVisitor < Crystal::SemanticVisitor
       scope.types[name] = type
     end
 
+    attach_doc type, node, annotations
+
     node.resolved_type = type
 
     type.private = true if node.visibility.private?
@@ -626,9 +628,7 @@ class Crystal::TopLevelVisitor < Crystal::SemanticVisitor
       type.extern = true
       type.extern_union = node.union?
 
-      if location = node.location
-        type.add_location(location)
-      end
+      attach_doc type, node, annotations
 
       current_type.types[node.name] = type
     end
@@ -641,15 +641,22 @@ class Crystal::TopLevelVisitor < Crystal::SemanticVisitor
   end
 
   def visit(node : TypeDef)
+    annotations = read_annotations
     type = current_type.types[node.name]?
+
     if type
       node.raise "#{node.name} is already defined"
     else
       typed_def_type = lookup_type(node.type_spec)
       typed_def_type = check_allowed_in_lib node.type_spec, typed_def_type
-      current_type.types[node.name] = TypeDefType.new @program, current_type, node.name, typed_def_type
-      false
+      type = TypeDefType.new @program, current_type, node.name, typed_def_type
+
+      attach_doc type, node, annotations
+
+      current_type.types[node.name] = type
     end
+
+    false
   end
 
   def visit(node : EnumDef)
@@ -1007,6 +1014,7 @@ class Crystal::TopLevelVisitor < Crystal::SemanticVisitor
     external.call_convention = call_convention
     external.varargs = node.varargs?
     external.fun_def = node
+    external.return_type = node.return_type
     node.external = external
 
     current_type.add_def(external)

src/compiler/crystal/semantic/type_declaration_visitor.cr

@@ -99,7 +99,27 @@ class Crystal::TypeDeclarationVisitor < Crystal::SemanticVisitor
     var_type = check_allowed_in_lib node.type_spec, var_type
 
     type = current_type.as(LibType)
-    type.add_var node.name, var_type, (node.real_name || node.name), thread_local
+
+    setter = External.new(
+      "#{node.name}=", [Arg.new("value", type: var_type)],
+      Primitive.new("external_var_set", var_type), node.real_name || node.name
+    ).at(node.location)
+    setter.set_type(var_type)
+    setter.external_var = true
+    setter.thread_local = thread_local
+    setter.doc = node.doc || @annotations.try(&.first?).try(&.doc)
+
+    getter = External.new(
+      "#{node.name}", [] of Arg,
+      Primitive.new("external_var_get", var_type), node.real_name || node.name
+    ).at(node.location)
+    getter.set_type(var_type)
+    getter.external_var = true
+    getter.thread_local = thread_local
+    getter.doc = node.doc || @annotations.try(&.first?).try(&.doc)
+
+    type.add_def setter
+    type.add_def getter
 
     false
   end
@@ -186,15 +206,25 @@ class Crystal::TypeDeclarationVisitor < Crystal::SemanticVisitor
     if type.lookup_instance_var?(var_name)
       node.raise "#{type.type_desc} #{type} already defines a field named '#{field_name}'"
     end
+
     ivar = MetaTypeVar.new(var_name, field_type)
+    ivar.doc = node.var.as(Var).doc
     ivar.owner = type
+
     declare_c_struct_or_union_field(type, field_name, ivar, node.location)
   end
 
   def declare_c_struct_or_union_field(type, field_name, var, location)
     type.instance_vars[var.name] = var
-    type.add_def Def.new("#{field_name}=", [Arg.new("value")], Primitive.new("struct_or_union_set").at(location))
-    type.add_def Def.new(field_name, body: InstanceVar.new(var.name))
+
+    setter = Def.new("#{field_name}=", [Arg.new("value")], Primitive.new("struct_or_union_set").at(location)).at(location)
+    setter.doc = var.doc
+
+    getter = Def.new(field_name, body: InstanceVar.new(var.name)).at(location)
+    getter.doc = var.doc
+
+    type.add_def setter
+    type.add_def getter
   end
 
   def declare_instance_var(node, var)

src/compiler/crystal/syntax/ast.cr

@@ -575,6 +575,7 @@ module Crystal
     include SpecialVar
 
     property name : String
+    property doc : String?
 
     def initialize(@name : String)
     end
@@ -1630,6 +1631,7 @@ module Crystal
   end
 
   class TypeDeclaration < ASTNode
+    property doc : String?
     property var : ASTNode
     property declared_type : ASTNode
     property value : ASTNode?
@@ -1925,6 +1927,7 @@ module Crystal
 
   class LibDef < ASTNode
     property name : Path
+    property doc : String?
     property body : ASTNode
     property name_location : Location?
     property visibility = Visibility::Public
@@ -1980,6 +1983,7 @@ module Crystal
 
   class TypeDef < ASTNode
     property name : String
+    property doc : String?
     property type_spec : ASTNode
     property name_location : Location?
 
@@ -2002,6 +2006,7 @@ module Crystal
   # A c struct/union definition inside a lib declaration
   class CStructOrUnionDef < ASTNode
     property name : String
+    property doc : String?
     property body : ASTNode
     property? union : Bool
 
@@ -2044,6 +2049,7 @@ module Crystal
 
   class ExternalVar < ASTNode
     property name : String
+    property doc : String?
     property type_spec : ASTNode
     property real_name : String?
 

src/compiler/crystal/syntax/parser.cr

@@ -5637,6 +5637,7 @@ module Crystal
     end
 
     def parse_lib
+      doc = @token.doc
       location = @token.location
       next_token_skip_space_or_newline
 
@@ -5652,6 +5653,7 @@ module Crystal
 
       lib_def = LibDef.new(name, body).at(location).at_end(end_location)
       lib_def.name_location = name_location
+      lib_def.doc = doc
       lib_def
     end
 
@@ -5708,6 +5710,7 @@ module Crystal
         skip_statement_end
         Assign.new(ident, value)
       when .global?
+        doc = @token.doc
         location = @token.location
         name = @token.value.to_s[1..-1]
         next_token_skip_space_or_newline
@@ -5727,6 +5730,7 @@ module Crystal
 
         skip_statement_end
         ExternalVar.new(name, type, real_name)
+          .at(location).tap(&.doc=(doc))
       when .op_lcurly_lcurly?
         parse_percent_macro_expression
       when .op_lcurly_percent?
@@ -5964,6 +5968,7 @@ module Crystal
     end
 
     def parse_type_def
+      doc = @token.doc
       next_token_skip_space_or_newline
       name = check_const
       name_location = @token.location
@@ -5976,10 +5981,13 @@ module Crystal
 
       typedef = TypeDef.new name, type
       typedef.name_location = name_location
+      typedef.doc = doc
+
       typedef
     end
 
     def parse_c_struct_or_union(union : Bool)
+      doc = @token.doc
       location = @token.location
       next_token_skip_space_or_newline
       name = check_const
@@ -5989,7 +5997,9 @@ module Crystal
       end_location = token_end_location
       next_token_skip_space
 
-      CStructOrUnionDef.new(name, Expressions.from(body), union: union).at(location).at_end(end_location)
+      CStructOrUnionDef.new(name, Expressions.from(body), union: union)
+        .at(location).at_end(end_location)
+        .tap(&.doc=(doc))
     end
 
     def parse_c_struct_or_union_body
@@ -6031,6 +6041,7 @@ module Crystal
     end
 
     def parse_c_struct_or_union_fields(exps)
+      doc = @token.doc
       vars = [Var.new(@token.value.to_s).at(@token.location).at_end(token_end_location)]
 
       next_token_skip_space_or_newline
@@ -6049,6 +6060,7 @@ module Crystal
       skip_statement_end
 
       vars.each do |var|
+        var.doc = doc
         exps << TypeDeclaration.new(var, type).at(var).at_end(type)
       end
     end

src/compiler/crystal/tools/doc/generator.cr

@@ -143,8 +143,10 @@ class Crystal::Doc::Generator
       return false if nodoc? ns
     end
 
-    # Don't include lib types or types inside a lib type
-    return false if type.is_a?(Crystal::LibType) || type.namespace.is_a?(LibType)
+    # Don't include lib types or types inside a lib type unless specified with `:showdoc:`
+    if (type.is_a?(LibType) || type.namespace.is_a?(LibType)) && !showdoc?(type)
+      return false
+    end
 
     !!type.locations.try &.any? do |type_location|
       must_include? type_location
@@ -262,7 +264,7 @@ class Crystal::Doc::Generator
 
     parent.types?.try &.each_value do |type|
       case type
-      when Const, LibType
+      when Const
         next
       else
         types << type(type) if must_include? type