Fix strip issue with 'nekotools boot' created executables on Linux ELF...

CHANGES

@@ -1,3 +1,11 @@
+2015-08-11 : ???
+	vm : added ability to find embedded bytecode anywhere within elf format binary; controlled by ABI_ELF define
+	std : added ability to inject bytecode into .nekobytecode section of elf format vm binary
+	perf : 'neko -version' execution slowed by 32% or ~4.4ms/execution on my 2.53Ghz Xeon
+	perf : 'nekoc -help' execution slowed by 4% or ~13.8ms/execution on my 2.53Ghz Xeon
+	size : neko vm grew 2,880 bytes or ~23% larger; still under 16KB
+	size : nekotools boot executables can be stripped now, making them 2%-8% smaller on x86_64
+
 2013-??-?? : 2.1.0
 	all : silent libs compilation by default
 	all : installation by default in /usr instead of /usr/local + some rights fixes

Makefile

@@ -15,7 +15,7 @@
 
 INSTALL_PREFIX = /usr
 
-CFLAGS = -Wall -O3 -fPIC -fomit-frame-pointer -I vm -D_GNU_SOURCE -I libs/common
+CFLAGS = -Wall -O3 -fPIC -fomit-frame-pointer -I vm -D_GNU_SOURCE -I libs/common -DABI_ELF
 EXTFLAGS = -pthread
 MAKESO = $(CC) -shared -Wl,-Bsymbolic
 LIBNEKO_NAME = libneko.so
@@ -77,8 +77,8 @@ endif
 ### MAKE
 
 VM_OBJECTS = vm/stats.o vm/main.o
-STD_OBJECTS = libs/std/buffer.o libs/std/date.o libs/std/file.o libs/std/init.o libs/std/int32.o libs/std/math.o libs/std/string.o libs/std/random.o libs/std/serialize.o libs/std/socket.o libs/std/sys.o libs/std/xml.o libs/std/module.o libs/common/sha1.o libs/std/md5.o libs/std/unicode.o libs/std/utf8.o libs/std/memory.o libs/std/misc.o libs/std/thread.o libs/std/process.o
-LIBNEKO_OBJECTS = vm/alloc.o vm/builtins.o vm/callback.o vm/interp.o vm/load.o vm/objtable.o vm/others.o vm/hash.o vm/module.o vm/jit_x86.o vm/threads.o
+STD_OBJECTS = libs/std/buffer.o libs/std/date.o libs/std/file.o libs/std/init.o libs/std/int32.o libs/std/math.o libs/std/string.o libs/std/random.o libs/std/serialize.o libs/std/socket.o libs/std/sys.o libs/std/xml.o libs/std/module.o libs/common/sha1.o libs/std/md5.o libs/std/unicode.o libs/std/utf8.o libs/std/memory.o libs/std/misc.o libs/std/thread.o libs/std/process.o libs/std/elf_update.o
+LIBNEKO_OBJECTS = vm/alloc.o vm/builtins.o vm/callback.o vm/elf.o vm/interp.o vm/load.o vm/objtable.o vm/others.o vm/hash.o vm/module.o vm/jit_x86.o vm/threads.o
 
 all: createbin libneko neko std compiler libs
 
@@ -90,11 +90,12 @@ libneko: bin/${LIBNEKO_NAME}
 libs:
 	(cd src; ${NEKO_EXEC} nekoc tools/install.neko)
 	(cd src; ${NEKO_EXEC} tools/install -silent ${INSTALL_FLAGS})
+	strip bin/nekoc bin/nekoml bin/nekotools
 
 tools:
 	(cd src; ${NEKO_EXEC} nekoc tools/install.neko)
 	(cd src; ${NEKO_EXEC} tools/install -nolibs)
-	
+
 doc:
 	(cd src; ${NEKO_EXEC} nekoc tools/makedoc.neko)
 	(cd src; ${NEKO_EXEC} tools/makedoc)

libs/std/elf_update.c

@@ -0,0 +1,87 @@
+/*
+ * Copyright (C)2015 Haxe Foundation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#include "neko_elf.h"
+
+
+static value elf_update_section_header_for_bytecode(value _file, value _interp_size, value _bytecode_size)
+{
+/* This function is a no-op on non-ELF platforms... */
+#ifdef SEPARATE_SECTION_FOR_BYTECODE
+
+	FILE *exe;
+        int interp_size, bytecode_size;
+	char buf[size_Shdr], *file;
+	int bytecode_sec_idx;
+
+	val_check(_file,string);
+	val_check(_interp_size,int);
+	val_check(_bytecode_size,int);
+	file = val_string(_file);
+	interp_size = val_int(_interp_size);
+	bytecode_size = val_int(_bytecode_size);
+
+	if ( interp_size%4 != 0 ) {
+		return val_false;
+	}
+
+	/* Open the file to update the elf nekobytecode section
+	   header... */
+	exe = fopen(file,"r+b");
+	if( exe == NULL )
+		return val_false;
+
+	/* First read the elf header... */
+	if ( val_true != elf_read_header(exe) ) goto failed;
+
+	/* Find the right section header... */
+	bytecode_sec_idx = elf_find_bytecode_section(exe);
+	if ( -1 == bytecode_sec_idx ) goto failed;
+
+	/* Now that we have the right section header, update it... */
+	if ( val_true != elf_read_section(exe,bytecode_sec_idx,buf) ) goto failed;
+
+	elf_set_Shdr(buf,sh_type,SHT_PROGBITS);
+	elf_set_Shdr(buf,sh_flags,elf_get_Shdr(buf,sh_flags) & (SHF_MASKOS|SHF_MASKPROC));
+	elf_set_Shdr(buf,sh_addr,0);
+	elf_set_Shdr(buf,sh_offset,interp_size);
+	elf_set_Shdr(buf,sh_size,bytecode_size);
+	elf_set_Shdr(buf,sh_addralign,1);
+	elf_set_Shdr(buf,sh_entsize,0);
+
+	/* ...and write it back... */
+	if ( val_true != elf_write_section(exe,bytecode_sec_idx,buf) ) goto failed;
+
+        elf_free_section_string_table();
+	fclose(exe);
+	return val_true;
+
+failed:
+        elf_free_section_string_table();
+	fclose(exe);
+	return val_false;
+#else
+	return val_true;
+#endif
+}
+
+DEFINE_PRIM(elf_update_section_header_for_bytecode,3);
+

libs/std/std.vcproj

@@ -173,6 +173,9 @@
 		<File
 			RelativePath=".\date.c">
 		</File>
+		<File
+			RelativePath=".\elf_update.c">
+		</File>
 		<File
 			RelativePath=".\file.c">
 		</File>

libs/std/std.vcxproj

@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
 <Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <ItemGroup Label="ProjectConfigurations">
     <ProjectConfiguration Include="Debug|Win32">
@@ -128,6 +128,7 @@
   <ItemGroup>
     <ClCompile Include="buffer.c" />
     <ClCompile Include="date.c" />
+    <ClCompile Include="elf_update.c" />
     <ClCompile Include="file.c" />
     <ClCompile Include="init.c" />
     <ClCompile Include="int32.c" />

src/tools/nekoboot.neko

@@ -22,6 +22,7 @@
 
 // primitives
 
+elf_update_section_header = $loader.loadprim("std@elf_update_section_header_for_bytecode",3);
 file_contents = $loader.loadprim("std@file_contents",1);
 file_open = $loader.loadprim("std@file_open",2);
 file_write = $loader.loadprim("std@file_write",4);
@@ -89,8 +90,15 @@ if( dot_pos != null )
 
 var out_name = file+exe_ext;
 var out = file_open(out_name,"wb");
+var bytecode_size = $ssize(bytecode);
+var pad_size = (4-(boot_size&0x3)) & 0x3;
+
 file_write(out,boot,0,boot_size);
-file_write(out,bytecode,0,$ssize(bytecode));
+boot_size += pad_size;
+if( pad_size >= 3 ) file_write_char(out,0x00);
+if( pad_size >= 2 ) file_write_char(out,0x00);
+if( pad_size >= 1 ) file_write_char(out,0x00);
+file_write(out,bytecode,0,bytecode_size)
 file_write(out,"NEKO",0,4);
 file_write_char(out,boot_size & 0xFF);
 file_write_char(out,(boot_size >> 8) & 0xFF);
@@ -105,3 +113,9 @@ switch system {
 	default => command("chmod 755 "+out_name)
 }
 
+// Update ELF section header (on platforms where that is appropriate) to protect
+// binary's bytecode from being removed by the strip program
+
+var res = elf_update_section_header(out_name,boot_size,bytecode_size+8);
+if( res == 0 )
+	$print("Trouble updating elf section header; stripping binary may lead to problems!")

vm/elf.c

@@ -0,0 +1,188 @@
+/*
+ * Copyright (C)2015 Haxe Foundation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+#include "neko_elf.h"
+
+/* None of this is needed on non-ELF platforms... */
+#ifdef SEPARATE_SECTION_FOR_BYTECODE
+
+#include <string.h>
+#include <stdlib.h>
+
+
+/* Name of neko bytecode section... */
+static const char const* BYTECODE_SEC_NAME = ".nekobytecode";
+
+/* Must be big enough to hold Elf32_Ehdr or Elf64_Ehdr... */
+int size_Ehdr = sizeof(Elf64_Ehdr);
+
+/* Must be big enough to hold Elf32_Shdr or Elf64_Shdr... */
+int size_Shdr = sizeof(Elf64_Shdr);
+
+
+static int is_32, shoff, shent, shnum, shstr;
+static char *strbuf;
+static int strsize, stroff;
+
+
+static value elf_read_exe(FILE *exe, int loc, char *buf, int size)
+{
+	if ( 0 != fseek(exe,loc,SEEK_SET) ||
+             size != fread(buf,1,size,exe) ) {
+		fclose(exe);
+		return val_false;
+	}
+	return val_true;
+}
+
+static value elf_write_exe(FILE *exe, int loc, char *buf, int size)
+{
+	if ( 0 != fseek(exe,loc,SEEK_SET) ||
+             size != fwrite(buf,1,size,exe) ) {
+		fclose(exe);
+		return val_false;
+	}
+	return val_true;
+}
+
+value elf_read_header(FILE *exe)
+{
+	char hdr[size_Ehdr];
+	int hdrsize;
+
+	/* First read the elf header to determine 32/64-bit-ness... */
+	if ( val_true != elf_read_exe(exe,0,hdr,EI_NIDENT) ) return val_false;
+	if ( hdr[EI_CLASS] == ELFCLASS32 || hdr[EI_CLASS] == ELFCLASS64 ) {
+		is_32 = hdr[EI_CLASS] == ELFCLASS32;
+	} else return val_false;
+
+	/* Read the full elf header now... */
+	hdrsize = is_32 ? sizeof(Elf32_Ehdr) : sizeof(Elf64_Ehdr);
+	if ( val_true != elf_read_exe(exe,0,hdr,hdrsize) ) return val_false;
+
+        if ( elf_get_Ehdr(hdr,e_type) != ET_EXEC ) return val_false;
+
+        /* Remember the section headers info... */
+	shoff = elf_get_Ehdr(hdr,e_shoff);
+	shent = elf_get_Ehdr(hdr,e_shentsize);
+	shnum = elf_get_Ehdr(hdr,e_shnum);
+	shstr = elf_get_Ehdr(hdr,e_shstrndx);
+
+        return val_true;
+}
+
+int elf_is_32()
+{
+	return is_32;
+}
+
+value elf_read_section(FILE *exe, int sec, char *buf)
+{
+	return elf_read_exe(exe,shoff+sec*shent,buf,shent);
+}
+
+value elf_write_section(FILE *exe, int sec, char *buf)
+{
+	return elf_write_exe(exe,shoff+sec*shent,buf,shent);
+}
+
+static value elf_read_section_string_table(FILE *exe)
+{
+	char buf[size_Ehdr];
+
+        if ( NULL != strbuf ) return val_true;
+
+	if ( val_true != elf_read_section(exe,shstr,buf) ) return val_false;
+	stroff  = elf_get_Shdr(buf,sh_offset);
+	strsize = elf_get_Shdr(buf,sh_size);
+	strbuf = (char*) malloc(strsize);
+	if ( val_true != elf_read_exe(exe,stroff,strbuf,strsize) ) return val_false;
+
+	return val_true;
+}
+
+void elf_free_section_string_table()
+{
+	if ( NULL != strbuf ) {
+		free(strbuf);
+                strbuf = NULL;
+	}
+}
+
+static int elf_find_section_by_name(FILE *exe, const char *name)
+{
+	char buf[size_Shdr];
+	int shcur = 0, shname;
+
+	if ( val_true != elf_read_section_string_table(exe) ) return -1;
+
+	while ( shcur < shnum ) {
+		if ( val_true != elf_read_section(exe,shcur,buf) ) return -1;
+		shname = elf_get_Shdr(buf,sh_name);
+		if ( shname < strsize && !strncmp(&strbuf[shname], name, strlen(name)) ) {
+			/* found the .nekobytecode section! */
+			return shcur;
+		}
+		shcur++;
+	}
+	return -1;
+}
+
+int elf_find_bytecode_section(FILE *exe)
+{
+	return elf_find_section_by_name(exe, BYTECODE_SEC_NAME);
+}
+
+value elf_find_embedded_bytecode(const char *file, int *beg, int *end)
+{
+	FILE *exe;
+	char buf[size_Shdr];
+	int bytecode_sec_idx;
+
+	/* Open the file to update the elf nekobytecode section
+	   header... */
+	exe = fopen(file,"rb");
+	if( exe == NULL )
+		return val_false;
+
+	/* First read the elf header... */
+	if ( val_true != elf_read_header(exe) ) goto failed;
+
+	/* Find the right section header... */
+	bytecode_sec_idx = elf_find_bytecode_section(exe);
+	if ( -1 == bytecode_sec_idx ) goto failed;
+
+	if ( val_true != elf_read_section(exe,bytecode_sec_idx,buf) ) goto failed;
+
+        elf_free_section_string_table();
+	fclose(exe);
+
+        if ( NULL != beg ) *beg = elf_get_Shdr(buf,sh_offset);
+        if ( NULL != end ) *end = elf_get_Shdr(buf,sh_offset) + elf_get_Shdr(buf,sh_size);
+        return val_true;
+
+failed:
+        elf_free_section_string_table();
+	fclose(exe);
+	return val_false;
+}
+
+#endif