Add tracy profiling to the SparrowCompiler

Add tracy repository as external
Add Profiling options for Sparrow. If enabled, one can capture profiling
data while running the compiler.
Add scopes for most important operations (including Nest_setContext,
Nest_computeType and Nest_semanticCheck).

Make the compiler used optimized version of the parser.

Author: lucteo

.gitmodules

@@ -2,3 +2,7 @@
 	path = externals/rapidcheck
 	url = https://github.com/emil-e/rapidcheck.git
 	branch = master
+[submodule "externals/tracy"]
+	path = externals/tracy
+	url = https://[email protected]/wolfpld/tracy.git
+	branch = master

CMakeLists.txt

@@ -48,6 +48,8 @@ CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/src/SparrowCompiler/VersionInfo.h.in
 # User passed compilation options
 option(BOOTSTRAP_SPARROW "Use system-wide SparrowCompiler to compile Sparrow files needed for the compiler" OFF)
 message(STATUS "BOOTSTRAP_SPARROW: ${BOOTSTRAP_SPARROW}")
+option(SPARROW_PROFILING "Enable Tracy integration into Sparrow compiler" OFF)
+message(STATUS "SPARROW_PROFILING: ${SPARROW_PROFILING}")
 
 # Where to output the results of the compilation
 set(OutDir ${CMAKE_CURRENT_SOURCE_DIR}/build/bin)
@@ -91,12 +93,18 @@ if(MSVC)
     add_definitions( -D_SCL_SECURE_NO_WARNINGS -D_CRT_SECURE_NO_DEPRECATE )
 else()
     add_definitions( -D__STDC_LIMIT_MACROS=1 )
-    set( CMAKE_CXX_FLAGS  "${CMAKE_CXX_FLAGS} -std=c++11" )
     # set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address" )
     add_definitions( -Wall )                    # All warnings...
     add_definitions( -Wno-deprecated )          # ... and except deprecated functions
 endif()
 
+if(SPARROW_PROFILING)
+    add_definitions( -DSPARROW_PROFILING=1 )
+    set( CMAKE_CXX_FLAGS  "${CMAKE_CXX_FLAGS} -std=c++17" )
+else()
+    set( CMAKE_CXX_FLAGS  "${CMAKE_CXX_FLAGS} -std=c++11" )
+endif()
+
 # Add our macros
 include(Macros.cmake)
 

Macros.cmake

@@ -8,6 +8,8 @@ set(CMAKE_INCLUDE_CURRENT_DIR ON)
 find_program(SPARROW_EXECUTABLE_EXT NAMES SparrowCompiler DOC "path to the SparrowCompiler executable (external)")
 # Try to find an external llc executable
 find_program(LLC_EXECUTABLE_EXT NAMES spr-llc "${LLVM_TOOLS_BINARY_DIR}/llc" DOC "path to the llc executable (external)")
+# Try to find an external opt executable
+find_program(OPT_EXECUTABLE_EXT NAMES spr-opt "${LLVM_TOOLS_BINARY_DIR}/opt" DOC "path to the opt executable (external)")
 
 MACRO(GROUP_NAME_FROM_PATH filePath resultVar)
     IF(MSVC OR APPLE)
@@ -104,7 +106,7 @@ macro(LLVMASM_TARGET Name Input Output)
         endif()
 
         add_custom_command(OUTPUT ${LLVMASM_TARGET_outputs}
-            COMMAND ${LLC_EXECUTABLE_EXT} --filetype=obj ${LLVMASM_EXECUTABLE_opts} -o ${Output} ${Input}
+            COMMAND ${OPT_EXECUTABLE_EXT} -O2 ${Input} | ${LLC_EXECUTABLE_EXT} --filetype=obj ${LLVMASM_EXECUTABLE_opts} -o ${Output}
             VERBATIM
             DEPENDS ${Input} ${LLVMASM_TARGET_ARG_DEPENDS}
             COMMENT "[LLVM][${Name}] Building object ${Output}"

externals/tracy

@@ -15,6 +15,7 @@
 #include "Nest/Api/Compiler.h"
 #include "Nest/Utils/CompilerSettings.hpp"
 #include "Nest/Utils/Diagnostic.hpp"
+#include "Nest/Utils/Profiling.h"
 #include "Nest/Utils/CompilerStats.hpp"
 #include "Nest/Utils/cppif/StringRef.hpp"
 
@@ -140,6 +141,7 @@ void CtModule::recreateModule() {
 }
 
 void CtModule::syncModule() {
+    PROFILING_ZONE();
     if (!llvmModule_->empty() || !llvmModule_->global_empty()) {
         // Uncomment this for debugging
         // llvmModule_->dump();
@@ -179,6 +181,8 @@ void CtModule::ctProcessBackendCode(Node* node) {
 }
 
 Node* CtModule::ctEvaluateExpression(Node* node) {
+    PROFILING_ZONE_TEXT(Nest_toStringEx(node));
+
     ASSERT(llvmModule_);
 
     // Gather statistics if requested
@@ -284,10 +288,17 @@ Node* CtModule::ctEvaluateExpression(Node* node) {
         //  - transform this into a function pointer that receives the output as a parameter
         //  - call the function, to fill up the data buffer
         using FunType = void (*)(const char*);
-        // NOLINTNEXTLINE(cppcoreguidelines-pro-type-cstyle-cast)
-        auto fptr = (FunType)llvmExecutionEngine_->getFunctionAddress(funName);
+        FunType fptr;
+        {
+            PROFILING_ZONE_NAMED("CT getFunctionAddress");
+            // NOLINTNEXTLINE(cppcoreguidelines-pro-type-cstyle-cast)
+            fptr = (FunType)llvmExecutionEngine_->getFunctionAddress(funName);
+        }
         ASSERT(fptr);
-        fptr(dataBuffer.begin);
+        {
+            PROFILING_ZONE_NAMED("CT exec");
+            fptr(dataBuffer.begin);
+        }
 
         // Create a CtValue containing the data resulted from expression evaluation
         Type t = node->type;
@@ -301,9 +312,17 @@ Node* CtModule::ctEvaluateExpression(Node* node) {
         //  - call the function
         using FunType = void (*)();
         // NOLINTNEXTLINE(cppcoreguidelines-pro-type-cstyle-cast)
-        auto fptr = (FunType)llvmExecutionEngine_->getFunctionAddress(funName);
+        FunType fptr;
+        {
+            PROFILING_ZONE_NAMED("CT getFunctionAddress");
+            // NOLINTNEXTLINE(cppcoreguidelines-pro-type-cstyle-cast)
+            fptr = (FunType)llvmExecutionEngine_->getFunctionAddress(funName);
+        }
         ASSERT(fptr);
-        fptr();
+        {
+            PROFILING_ZONE_NAMED("CT exec");
+            fptr();
+        }
 
         // Create a Nop operation for return
         res = Feather_mkNop(node->location);

src/LLVMBackend/CtModule.cpp

@@ -6,6 +6,7 @@
 #include "Nest/Api/Compiler.h"
 #include "Nest/Utils/CompilerSettings.hpp"
 #include "Nest/Utils/CompilerStats.hpp"
+#include "Nest/Utils/Profiling.h"
 
 #include <boost/filesystem.hpp>
 
@@ -49,6 +50,13 @@ string replaceExtension(
 /// Run the command with the given arguments
 /// The command should be the first in the list of arguments
 void runCmd(const vector<string>& args) {
+#if SPARROW_PROFILING
+    std::ostringstream ossProfiling;
+    for (const string& arg : args)
+        ossProfiling << arg << " ";
+    PROFILING_ZONE_TEXT(ossProfiling.str().c_str());
+#endif
+
     ASSERT(args.size() > 0);
 
     const auto& s = *Nest_compilerSettings();
@@ -84,6 +92,8 @@ void runCmd(const vector<string>& args) {
 
 /// Write the given LLVM module, as a bitcode to disk
 void writeBitcodeFile(const Module& module, const string& outputFilename) {
+    PROFILING_ZONE();
+
     error_code errorInfo;
     unique_ptr<TOOL_OUPUT_FILE_CLS> outFile(
             new TOOL_OUPUT_FILE_CLS(outputFilename.c_str(), errorInfo, sys::fs::OpenFlags::F_None));
@@ -98,6 +108,8 @@ void writeBitcodeFile(const Module& module, const string& outputFilename) {
 
 /// Write the given LLVM module, as an assembly file to disk
 void writeAssemblyFile(const Module& module, const string& outputFilename) {
+    PROFILING_ZONE();
+
     error_code errorInfo;
     unique_ptr<TOOL_OUPUT_FILE_CLS> outFile(
             new TOOL_OUPUT_FILE_CLS(outputFilename.c_str(), errorInfo, sys::fs::OpenFlags::F_None));
@@ -185,13 +197,17 @@ void generateNativeObjGCC(
 } // namespace
 
 void LLVMB::generateRtAssembly(const llvm::Module& module) {
+    PROFILING_ZONE();
+
     const auto& s = *Nest_compilerSettings();
 
     string filename = replaceExtension(s.output_, s.output_, ".ll");
     writeAssemblyFile(module, filename);
 }
 
 void LLVMB::generateCtAssembly(const llvm::Module& module) {
+    PROFILING_ZONE();
+
     const auto& s = *Nest_compilerSettings();
 
     // Safety check
@@ -229,22 +245,30 @@ void LLVMB::link(const vector<llvm::Module*>& inputs, const string& outFilename)
     // the sourcecodes into one big module, and perform those operations here
 
     // Link all the input modules to a single module
-    // we desotry all the modules in this process
+    // we destroy all the modules in this process
     if (inputs.empty())
         REP_INTERNAL(NOLOC, "At least one bitcode needs to be passed to the linker");
     llvm::Module* compositeModule = inputs[0];
-    llvm::Linker liner(*compositeModule);
-    for (size_t i = 1; i < inputs.size(); ++i) {
-        unique_ptr<llvm::Module> mod(inputs[i]);
-        if (liner.linkInModule(move(mod), llvm::Linker::OverrideFromSrc))
-            REP_INTERNAL(NOLOC, "Link error");
+    {
+        PROFILING_ZONE_NAMED("linking modules")
+
+        llvm::Linker linker(*compositeModule);
+        for (size_t i = 1; i < inputs.size(); ++i) {
+            unique_ptr<llvm::Module> mod(inputs[i]);
+            if (linker.linkInModule(move(mod), llvm::Linker::OverrideFromSrc))
+                REP_INTERNAL(NOLOC, "Link error");
+        }
     }
 
     // Verify the module
-    string err;
-    raw_string_ostream errStream(err);
-    if (verifyModule(*compositeModule, &errStream))
-        REP_INTERNAL(NOLOC, "LLVM Verification failed for generated program: %1%") % err;
+    {
+        PROFILING_ZONE_NAMED("verifying composite module")
+
+        string err;
+        raw_string_ostream errStream(err);
+        if (verifyModule(*compositeModule, &errStream))
+            REP_INTERNAL(NOLOC, "LLVM Verification failed for generated program: %1%") % err;
+    }
 
     bool shouldOptimize = !s.optimizerArgs_.empty() || s.optimizationLevel_ != "0";
 

src/LLVMBackend/Generator.cpp

@@ -8,6 +8,7 @@
 #include <Nest/Api/SourceCode.h>
 #include <Nest/Utils/cppif/NodeUtils.hpp>
 #include <Nest/Utils/Diagnostic.hpp>
+#include "Nest/Utils/Profiling.h"
 #include <Feather/Api/Feather.h>
 
 using namespace LLVMB;
@@ -28,6 +29,8 @@ bool isDecl(Node* node) {
 } // namespace
 
 void Tr::prepareTranslate(Node* node, GlobalContext& ctx) {
+    // PROFILING_ZONE();
+
     if (!node)
         return;
 

src/LLVMBackend/Tr/PrepareTranslate.cpp

@@ -1,6 +1,7 @@
 #pragma once
 
 #include "TypeRef.h"
+#include "Nest/Utils/ProfilingFwd.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -33,6 +34,14 @@ FToString Nest_getToStringFun(int nodeKind);
 //! Resets the registered node kinds
 void Nest_resetRegisteredNodeKinds();
 
+#if SPARROW_PROFILING
+
+const Nest_Profiling_LocType* Nest_Profiling_getSetContextLoc(int nodeKind);
+const Nest_Profiling_LocType* Nest_Profiling_getComputeTypeLoc(int nodeKind);
+const Nest_Profiling_LocType* Nest_Profiling_getSemanticCheckLoc(int nodeKind);
+
+#endif
+
 #ifdef __cplusplus
 }
 #endif

src/Nest/Api/NodeKindRegistrar.h

@@ -69,7 +69,15 @@ SET(sourceFiles
     "src/Utils/cppif/StringRef.cpp"
     "src/Utils/cppif/Type.cpp"
     "src/Utils/cppif/TypeWithStorage.cpp"
+
 )
+if(SPARROW_PROFILING)
+    SET(sourceFiles
+        ${sourceFiles}
+        "src/Utils/Profiling.cpp"
+    )
+    add_definitions( -DSPARROW_PROFILING=1 )
+endif()
 
 # Project settings
 INCLUDE_DIRECTORIES( "." )

src/Nest/CMakeLists.txt

@@ -0,0 +1,104 @@
+#pragma once
+
+#include "ProfilingFwd.h"
+
+#if SPARROW_PROFILING
+
+#define TRACY_ENABLE 1
+
+#ifdef __cplusplus
+
+////////////////////////////////////////////////////////////////////////////////
+// C++ profiling definitions
+
+#include "../../../externals/tracy/Tracy.hpp"
+
+#include "Nest/Api/StringRef.h"
+
+#define PROFILING_ZONE() ZoneScoped
+#define PROFILING_ZONE_NAMED(staticName) ZoneScopedN(staticName)
+#define PROFILING_ZONE_TEXT(text)                                                                  \
+    ZoneScoped;                                                                                    \
+    _Nest_Profiling_zoneSetText(___tracy_scoped_zone, text)
+#define PROFILING_ZONE_NAMED_TEXT(staticName, text)                                                \
+    ZoneScopedN(staticName);                                                                       \
+    _Nest_Profiling_zoneSetText(___tracy_scoped_zone, text)
+#define PROFILING_ZONE_SETTEEXT(text) _Nest_Profiling_zoneSetText(___tracy_scoped_zone, text)
+
+#define PROFILING_PLOT(name, val) tracy::Profiler::PlotData(name, val)
+
+#define PROFILING_MESSAGE_STATIC(staticText) tracy::Profiler::Message(staticText)
+#define PROFILING_MESSAGE(text) _Nest_Profiling_message(text)
+
+inline void _Nest_Profiling_zoneSetText(tracy::ScopedZone& zone, const char* text) {
+    zone.Text(text, strlen(text));
+}
+inline void _Nest_Profiling_zoneSetText(tracy::ScopedZone& zone, Nest_StringRef text) {
+    zone.Text(text.begin, text.end - text.begin);
+}
+
+inline void _Nest_Profiling_message(const char* text) {
+    tracy::Profiler::Message(text, strlen(text));
+}
+inline void _Nest_Profiling_message(Nest_StringRef text) {
+    tracy::Profiler::Message(text.begin, text.end - text.begin);
+}
+
+#else // __cplusplus
+
+////////////////////////////////////////////////////////////////////////////////
+// C-only profiling definitions
+
+#include "../../../externals/tracy/TracyC.h"
+
+typedef TracyCZoneCtx Nest_Profiling_ZoneCtx;
+
+#define PROFILING_C_ZONE_BEGIN(ctx)                                                                \
+    static const struct Nest_Profiling_LocType TracyConcat(__tracy_source_location, __LINE__) = {  \
+            NULL, __FUNCTION__, __FILE__, (uint32_t)__LINE__, 0};                                  \
+    Nest_Profiling_ZoneCtx ctx =                                                                   \
+            ___tracy_emit_zone_begin(&TracyConcat(__tracy_source_location, __LINE__), 1);
+
+#define PROFILING_C_ZONE_BEGIN_LOC(ctx, locPtr)                                                    \
+    Nest_Profiling_ZoneCtx ctx = ___tracy_emit_zone_begin(locPtr, 1)
+
+#define PROFILING_C_ZONE_END(ctx) ___tracy_emit_zone_end(ctx)
+#define PROFILING_C_ZONE_SETTEEXT(ctx, text) ___tracy_emit_zone_text(ctx, text, strlen(text))
+
+#endif // __cplusplus
+
+////////////////////////////////////////////////////////////////////////////////
+// Both C and C++ profiling definitions
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+//! Creates a location type for profiling
+const Nest_Profiling_LocType* Nest_Profiling_createLoc(
+        const char* name, const char* function, const char* file, unsigned line, unsigned color);
+
+#ifdef __cplusplus
+}
+#endif
+
+#else // SPARROW_PROFILING
+
+////////////////////////////////////////////////////////////////////////////////
+// Profiling not enabled
+
+#define PROFILING_ZONE()                            /*nothing*/
+#define PROFILING_ZONE_NAMED(staticName)            /*nothing*/
+#define PROFILING_ZONE_TEXT(text)                   /*nothing*/
+#define PROFILING_ZONE_NAMED_TEXT(staticName, text) /*nothing*/
+#define PROFILING_ZONE_SETTEEXT(text)               /*nothing*/
+#define PROFILING_PLOT(name, val)                   /*nothing*/
+#define PROFILING_MESSAGE_STATIC(staticText)        /*nothing*/
+#define PROFILING_MESSAGE(text)                     /*nothing*/
+
+#define PROFILING_C_ZONE_BEGIN(ctx)             /*nothing*/
+#define PROFILING_C_ZONE_BEGIN_LOC(ctx, locPtr) /*nothing*/
+#define PROFILING_C_ZONE_END(ctx)               /*nothing*/
+#define PROFILING_C_ZONE_SETTEEXT(ctx, text)    /*nothing*/
+
+#endif