Merge pull request #140 from onyx-lang/rc-v0.1.11

Release: 0.1.11

Author: brendanfh

CHANGELOG

@@ -1,20 +1,38 @@
 Release 0.1.11
 --------------
-Not released yet
+21st April 2024
 
 Additions:
 - Ability specify where piped arguments are placed using `_`.
     - `x |> foo(y, _) == foo(y, x)`
+- Alternative syntax for `case #default ...`. You can now just write `case _ ...`.
+- Alternative syntax for binding documentation using `///`.
+- **Experimental** compiler extensions feature, currently used to create procedural macros.
+- `core.misc.any_deep_copy`
+- Ability to explicitly specify tag value for tagged unions.
+    - `Variant as value: type`, i.e. `Foo as 3: i32`
 
 Removals:
+- Deprecated the use of `#default` in case statements. Use `_` instead.
+- Removed `iter.take_one`. Use `iter.next` instead.
 
 Changes:
+There are several *breaking* changes in this release related to core library APIs.
+- `Iterator.next` now returns `? T` instead of `(T, bool)`
+- `io.Stream` uses `Result(T, Error)` for return types instead of `(Error, T)`
+- `switch` over a `range` is no longer inclusive by default, since `..=` exists now.
+- Enabled optional semicolons by default.
+    - `//+optional-semicolons` is no longer necessary.
+
+There are also several non-breaking changes.
+- The internal memory layout is different. See pull request #133 for details.
 
 Bugfixes:
 
 Contributors:
 
 
+
 Release 0.1.10
 ---------------
 30th March 2024

build.bat

@@ -1,7 +1,7 @@
 @echo off
 
 REM Compile the compiler
-set SOURCE_FILES=compiler/src/onyx.c compiler/src/astnodes.c compiler/src/builtins.c compiler/src/checker.c compiler/src/clone.c compiler/src/doc.c compiler/src/entities.c compiler/src/errors.c compiler/src/lex.c compiler/src/parser.c compiler/src/symres.c compiler/src/types.c compiler/src/utils.c compiler/src/wasm_emit.c compiler/src/wasm_runtime.c
+set SOURCE_FILES=compiler/src/onyx.c compiler/src/astnodes.c compiler/src/builtins.c compiler/src/checker.c compiler/src/clone.c compiler/src/doc.c compiler/src/entities.c compiler/src/errors.c compiler/src/lex.c compiler/src/parser.c compiler/src/symres.c compiler/src/types.c compiler/src/utils.c compiler/src/wasm_emit.c compiler/src/wasm_runtime.c compiler/src/extensions.c
 
 if "%1" == "1" (
     set FLAGS=/Od /MTd /Z7

compiler/build.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-C_FILES="onyx astnodes builtins checker clone doc entities errors lex parser symres types utils wasm_emit "
+C_FILES="onyx astnodes builtins checker clone doc entities errors lex parser symres types utils wasm_emit extensions "
 LIBS="-lpthread -ldl -lm"
 INCLUDES="-I./include -I../shared/include -I../shared/include/dyncall"
 

compiler/include/astnodes.h

@@ -3,7 +3,7 @@
 
 #define VERSION_MAJOR 0
 #define VERSION_MINOR 1
-#define VERSION_PATCH 10
+#define VERSION_PATCH 11
 
 #include "stb_ds.h"
 #include "lex.h"
@@ -120,7 +120,11 @@
                                \
     NODE(ZeroValue)            \
                                \
-    NODE(JsNode)               
+    NODE(JsNode)               \
+                               \
+    NODE(CompilerExtension)    \
+    NODE(ProceduralMacro)      \
+    NODE(ProceduralExpansion)
 
 #define NODE(name) typedef struct Ast ## name Ast ## name;
 AST_NODES
@@ -255,6 +259,10 @@ typedef enum AstKind {
 
     Ast_Kind_Js_Code,
 
+    Ast_Kind_Compiler_Extension,
+    Ast_Kind_Procedural_Macro,
+    Ast_Kind_Procedural_Expansion,
+
     Ast_Kind_Count
 } AstKind;
 
@@ -1097,6 +1105,8 @@ struct AstUnionType {
 struct AstUnionVariant {
     AstTyped_base;
     bh_arr(AstTyped *) meta_tags;
+
+    AstTyped *explicit_tag_value;
 };
 struct AstPolyUnionType {
     AstType_base;
@@ -1635,6 +1645,33 @@ struct AstJsNode {
 };
 
 
+struct AstCompilerExtension {
+    AstNode_base;
+
+    OnyxToken *name;
+    bh_arr(AstProceduralMacro *) proc_macros;
+
+    i32 extension_id;
+};
+
+struct AstProceduralMacro {
+    AstNode_base;
+
+    // name is stored in the `token`
+
+    AstCompilerExtension *extension;
+};
+
+struct AstProceduralExpansion {
+    AstTyped_base;
+
+    AstTyped *proc_macro;
+    OnyxToken *expansion_body;
+
+    u32 expansion_id;
+};
+
+
 typedef struct EntityJobData {
     enum TypeMatch (*func)(void *job_data);
     void *job_data;
@@ -1671,6 +1708,8 @@ typedef enum EntityType {
     Entity_Type_Static_If,
     Entity_Type_String_Literal,
     Entity_Type_File_Contents,
+    Entity_Type_Compiler_Extension,
+    Entity_Type_Procedural_Expansion,
     Entity_Type_Enum,
     Entity_Type_Enum_Value,
     Entity_Type_Type_Alias,
@@ -1744,6 +1783,8 @@ typedef struct Entity {
         AstDirectiveLibrary   *library;
         EntityJobData         *job_data;
         AstJsNode             *js;
+        AstCompilerExtension  *compiler_extension;
+        AstProceduralExpansion *proc_expansion;
     };
 } Entity;
 
@@ -1865,6 +1906,34 @@ typedef struct DefinedVariable {
 } DefinedVariable;
 
 
+typedef enum ProceduralMacroExpansionKind {
+    PMEK_Expression,
+    PMEK_Statement,
+    PMEK_Top_Level
+} ProceduralMacroExpansionKind;
+
+typedef enum CompilerExtensionState {
+    COMP_EXT_STATE_SPAWNING,
+    COMP_EXT_STATE_INITIATING,
+    COMP_EXT_STATE_READY,
+    COMP_EXT_STATE_EXPANDING,
+} CompilerExtensionState;
+
+typedef struct CompilerExtension {
+    u64 pid;
+    u64 send_file;
+    u64 recv_file;
+
+    char *name;
+
+    i32 current_expansion_id;
+    CompilerExtensionState state;
+
+    bh_arena arena;
+
+    b32 alive : 1;
+} CompilerExtension;
+
 typedef struct CompileOptions CompileOptions;
 struct CompileOptions {
     bh_allocator allocator;
@@ -1876,6 +1945,7 @@ struct CompileOptions {
     b32 print_static_if_results : 1;
     b32 no_colors               : 1;
     b32 no_file_contents        : 1;
+    b32 no_compiler_extensions  : 1;
 
     b32 use_post_mvp_features : 1;
     b32 use_multi_threading   : 1;
@@ -1944,6 +2014,9 @@ struct Context {
     struct SymbolInfoTable *symbol_info;
     struct OnyxDocInfo     *doc_info;
 
+    bh_arr(CompilerExtension) extensions;
+    u32 next_expansion_id;
+
     CheckerData checker;
     ContextCaches caches;
     OnyxErrors errors;
@@ -2175,6 +2248,20 @@ void track_declaration_for_symbol_info(OnyxFilePos, AstNode *);
 void track_documentation_for_symbol_info(AstNode *, AstBinding *);
 void track_resolution_for_symbol_info(AstNode *original, AstNode *resolved);
 
+
+// Compiler Extensions
+TypeMatch compiler_extension_start(const char *name, const char *containing_filename, i32 *out_extension_id);
+TypeMatch compiler_extension_expand_macro(
+    int extension_id,
+    ProceduralMacroExpansionKind kind,
+    const char *macro_name,
+    OnyxToken *body,
+    Entity *entity,
+    AstNode **out_node,
+    u32 *out_expansion_id,
+    b32 wait_for_response);
+
+
 // NOTE: Useful inlined functions
 static inline b32 is_lval(AstNode* node) {
     node = strip_aliases(node);

compiler/include/lex.h

@@ -89,6 +89,8 @@ typedef enum TokenType {
 
     Token_Type_Doc_Comment,
 
+    Token_Type_Proc_Macro_Body,
+
     Token_Type_Count,
 } TokenType;
 

compiler/include/parser.h

@@ -70,5 +70,8 @@ void* onyx_ast_node_new(bh_allocator alloc, i32 size, AstKind kind);
 OnyxParser onyx_parser_create(bh_allocator alloc, OnyxTokenizer *tokenizer);
 void onyx_parser_free(OnyxParser* parser);
 void onyx_parse(OnyxParser *parser);
+AstTyped *onyx_parse_expression(OnyxParser *parser, Scope *scope);
+AstNode  *onyx_parse_statement(OnyxParser *parser, Scope *scope);
+void onyx_parse_top_level_statements(OnyxParser *parser, Scope *scope);
 
 #endif // #ifndef ONYXPARSER_H

compiler/src/astnodes.c

@@ -119,6 +119,10 @@ static const char* ast_node_names[] = {
 
     "JS CODE",
 
+    "COMPILER EXTENSION",
+    "PROCEDURAL MACRO",
+    "PROCEDURAL EXPANSION",
+
     "AST_NODE_KIND_COUNT",
 };
 
@@ -167,6 +171,8 @@ const char* entity_type_strings[Entity_Type_Count] = {
     "Static If",
     "String Literal",
     "File Contents",
+    "CompilerExtension",
+    "Procedural Expansion",
     "Enum",
     "Enum Value",
     "Type Alias",
@@ -1469,6 +1475,8 @@ b32 cast_is_legal(Type* from_, Type* to_, char** err_msg) {
 TypeMatch implicit_cast_to_bool(AstTyped **pnode) {
     AstTyped *node = *pnode;
 
+    if (!node->type) return TYPE_MATCH_YIELD;
+
     if ((node->type->kind == Type_Kind_Basic && node->type->Basic.kind == Basic_Kind_Rawptr)
         || (node->type->kind == Type_Kind_Pointer)
         || (node->type->kind == Type_Kind_MultiPointer)) {

compiler/src/checker.c

@@ -398,6 +398,7 @@ CheckStatus check_for(AstFor* fornode) {
         // HACK: This assumes the Iterator type only has a single type argument.
         given_type = iter_type->Struct.poly_sln[0].type;
         fornode->loop_type = For_Loop_Iterator;
+        fornode->var->flags |= Ast_Flag_Address_Taken;
     }
 
     if (given_type == NULL)
@@ -612,8 +613,7 @@ CheckStatus check_switch(AstSwitch* switchnode) {
                 i64 lower = ((AstNumLit *) rl->low)->value.l;
                 i64 upper = ((AstNumLit *) rl->high)->value.l;
 
-                // NOTE: This is inclusive!!!!
-                fori (case_value, lower, upper + 1) {
+                fori (case_value, lower, upper) {
                     if (add_case_to_switch_statement(switchnode, case_value, sc, rl->token->pos))
                         return Check_Error;
                 }
@@ -930,7 +930,10 @@ CheckStatus check_call(AstCall** pcall) {
 
     char* err_msg = NULL;
     fill_in_arguments(&call->args, (AstNode *) callee, &err_msg, 0);
-    if (err_msg != NULL) ERROR(call->token->pos, err_msg);
+    if (err_msg != NULL) {
+        onyx_report_error(callee->token->pos, Error_Critical, "Here is the function being called.");
+        ERROR(call->token->pos, err_msg);
+    }
 
     bh_arr(AstArgument *) arg_arr = (bh_arr(AstArgument *)) call->args.values;
     bh_arr_each(AstArgument *, arg, arg_arr) {
@@ -2653,6 +2656,8 @@ CheckStatus check_expression(AstTyped** pexpr) {
         case Ast_Kind_Foreign_Block: break;
         case Ast_Kind_Zero_Value: break;
         case Ast_Kind_Interface: break;
+        case Ast_Kind_Compiler_Extension: break;
+        case Ast_Kind_Procedural_Macro: break;
 
         default:
             retval = Check_Error;
@@ -3360,6 +3365,9 @@ CheckStatus check_union(AstUnionType *u_node) {
     bh_arr_each(AstUnionVariant *, variant, u_node->variants) {
         CHECK(type, &(* variant)->type_node);
         CHECK(meta_tags, (* variant)->meta_tags);
+        if ((*variant)->explicit_tag_value) {
+            CHECK(expression, &(* variant)->explicit_tag_value);
+        }
     }
 
     type_build_from_ast(context.ast_alloc, (AstType *) u_node);

compiler/src/cli.c

@@ -94,6 +94,7 @@ static const char *build_docstring = DOCSTRING_HEADER
     C_LBLUE "    --print-static-if-results   " C_NORM "Prints the conditional result of every " C_YELLOW "#if" C_NORM " statement\n"
     "\n"
     C_LBLUE "    --no-file-contents          " C_NORM "Disables " C_YELLOW "#file_contents" C_NORM " for security\n"
+    C_LBLUE "    --no-compiler-extensions    " C_NORM "Disables " C_YELLOW "#compiler_extension" C_NORM " for security\n"
     "\n";
 
 static const char *self_upgrade_docstring = DOCSTRING_HEADER
@@ -271,6 +272,9 @@ static void cli_parse_compilation_options(CompileOptions *options, int arg_parse
         else if (!strcmp(argv[i], "--no-file-contents")) {
             options->no_file_contents = 1;
         }
+        else if (!strcmp(argv[i], "--no-compiler-extensions")) {
+            options->no_compiler_extensions = 1;
+        }
         else if (!strcmp(argv[i], "--wasm-mvp")) {
             options->use_post_mvp_features = 0;
         }
@@ -429,7 +433,7 @@ static CompileOptions compile_opts_parse(bh_allocator alloc, int argc, char *arg
         .no_stale_code           = 0,
         .show_all_errors         = 0,
 
-        .enable_optional_semicolons = 0,
+        .enable_optional_semicolons = 1,
 
         .runtime = Runtime_Onyx,
 
@@ -501,7 +505,6 @@ static CompileOptions compile_opts_parse(bh_allocator alloc, int argc, char *arg
         case ONYX_COMPILE_ACTION_RUN:
         case ONYX_COMPILE_ACTION_WATCH:
         case ONYX_COMPILE_ACTION_COMPILE:
-        case ONYX_COMPILE_ACTION_RUN_WASM:
             cli_parse_compilation_options(&options, arg_parse_start, argc, argv);
             break;
 
@@ -515,6 +518,7 @@ static CompileOptions compile_opts_parse(bh_allocator alloc, int argc, char *arg
             }
             break;
 
+        case ONYX_COMPILE_ACTION_RUN_WASM:
         case ONYX_COMPILE_ACTION_PRINT_HELP:
         case ONYX_COMPILE_ACTION_PRINT_VERSION:
         case ONYX_COMPILE_ACTION_DOCUMENT:

compiler/src/entities.c

@@ -431,6 +431,22 @@ void add_entities_for_node(bh_arr(Entity *) *target_arr, AstNode* node, Scope* s
             break;
         }
 
+        case Ast_Kind_Compiler_Extension: {
+            ent.type = Entity_Type_Compiler_Extension;
+            ent.compiler_extension = (AstCompilerExtension *) node;
+            ent.state = Entity_State_Resolve_Symbols;
+            ENTITY_INSERT(ent);
+            break;
+        }
+
+        case Ast_Kind_Procedural_Expansion: {
+            ent.type = Entity_Type_Procedural_Expansion;
+            ent.proc_expansion = (AstProceduralExpansion *) node;
+            ent.state = Entity_State_Resolve_Symbols;
+            ENTITY_INSERT(ent);
+            break;
+        }
+
         default: {
             ent.type = Entity_Type_Expression;
             ent.expr = (AstTyped *) node;