Merge branch 'jqlang:master' into master

Author: sachintu47

.github/workflows/ci.yml

@@ -88,7 +88,6 @@ jobs:
           ./configure \
             --host=${{ matrix.CC }} \
             --disable-docs \
-            --disable-valgrind \
             --with-oniguruma=builtin \
             --enable-static \
             --enable-all-static \
@@ -154,7 +153,6 @@ jobs:
           ./configure \
             --host="${{ matrix.target }}$(uname -r)" \
             --disable-docs \
-            --disable-valgrind \
             --with-oniguruma=builtin \
             --enable-static \
             --enable-all-static \
@@ -230,7 +228,6 @@ jobs:
           autoreconf -i
           ./configure \
             --disable-docs \
-            --disable-valgrind \
             --with-oniguruma=builtin \
             --disable-shared \
             --enable-static \
@@ -276,7 +273,6 @@ jobs:
           autoreconf -i
           ./configure \
             --disable-docs \
-            --disable-valgrind \
             --with-oniguruma=builtin
           make distcheck
           make dist dist-zip

.github/workflows/oniguruma.yml

@@ -22,6 +22,7 @@ jobs:
           autoreconf -i
           ./configure \
             --disable-docs \
+            --enable-valgrind \
             --with-oniguruma=yes
           make -j"$(nproc)"
           file ./jq
@@ -54,6 +55,7 @@ jobs:
           autoreconf -i
           ./configure \
             --disable-docs \
+            --enable-valgrind \
             --with-oniguruma=no
           make -j"$(nproc)"
           file ./jq

.github/workflows/scanbuild.yml

@@ -49,7 +49,7 @@ jobs:
                 MAKEVARS: ${{ matrix.makevars }}
               run: |
                 autoreconf -i
-                ./configure --with-oniguruma=builtin $COVERAGE
+                ./configure --with-oniguruma=builtin --enable-valgrind $COVERAGE
                 scan-build --keep-going make -j4
             - name: Test
               env:

.github/workflows/valgrind.yml

@@ -22,6 +22,7 @@ jobs:
           autoreconf -i
           ./configure \
             --disable-docs \
+            --enable-valgrind \
             --with-oniguruma=builtin
           make -j"$(nproc)"
           file ./jq

Dockerfile

@@ -19,7 +19,6 @@ COPY . /app
 RUN autoreconf -i \
  && ./configure \
       --disable-docs \
-      --disable-valgrind \
       --with-oniguruma=builtin \
       --enable-static \
       --enable-all-static \

Makefile.am

@@ -65,23 +65,18 @@ endif
 
 include_HEADERS = src/jv.h src/jq.h
 
-if ENABLE_UBSAN
-AM_CFLAGS += -fsanitize=undefined
-endif
-
 AM_CPPFLAGS = -I$(srcdir)/src -I$(srcdir)/vendor
 
-### Running tests under Valgrind
+### Address sanitizer (ASan)
 
 if ENABLE_ASAN
 AM_CFLAGS += -fsanitize=address
-NO_VALGRIND = 1
-else
-if ENABLE_VALGRIND
-NO_VALGRIND =
-else
-NO_VALGRIND = 1
 endif
+
+### Undefined Behavior Sanitizer
+
+if ENABLE_UBSAN
+AM_CFLAGS += -fsanitize=undefined
 endif
 
 ### Code coverage with gcov
@@ -127,7 +122,11 @@ TESTS = tests/mantest tests/jqtest tests/shtest tests/utf8test tests/base64test
 if !WIN32
 TESTS += tests/optionaltest
 endif
-AM_TESTS_ENVIRONMENT = JQ=$(abs_builddir)/jq NO_VALGRIND=$(NO_VALGRIND)
+
+AM_TESTS_ENVIRONMENT = JQ=$(abs_builddir)/jq
+if ENABLE_VALGRIND
+AM_TESTS_ENVIRONMENT += ENABLE_VALGRIND=1
+endif
 
 # This is a magic make variable that causes it to treat tests/man.test as a
 # DATA-type dependency for the check target. As a result, it will attempt to

configure.ac

@@ -39,11 +39,6 @@ if test "$USE_MAINTAINER_MODE" = yes; then
   AC_CHECK_PROGS(LEX, flex lex)
 fi
 
-dnl Check for valgrind
-AC_CHECK_PROGS(valgrind_cmd, valgrind)
-if test "x$valgrind_cmd" = "x" ; then
-    AC_MSG_WARN([valgrind is required to test jq.])
-fi
 AC_CHECK_FUNCS(memmem)
 
 AC_CHECK_HEADER("sys/cygwin.h", [have_cygwin=1;])
@@ -54,10 +49,9 @@ dnl Running tests with Valgrind is slow. It is faster to iterate on
 dnl code without Valgrind until tests pass, then enable Valgrind and
 dnl fix leaks.
 AC_ARG_ENABLE([valgrind],
-   AS_HELP_STRING([--disable-valgrind],[do not run tests under Valgrind]))
+   AS_HELP_STRING([--enable-valgrind],[enable Valgrind during testing]))
 
-dnl Running tests with Valgrind is slow; address sanitizer (ASAN) is
-dnl faster.
+dnl Address sanitizer (ASan)
 AC_ARG_ENABLE([asan],
    AS_HELP_STRING([--enable-asan],[enable address sanitizer]))
 
@@ -114,7 +108,7 @@ AS_IF([test "x$enable_decnum" != "xno"],[
    AC_DEFINE([USE_DECNUM], 1, [Define to enable decnum support.])
 ])
 
-AM_CONDITIONAL([ENABLE_VALGRIND], [test "x$enable_valgrind" != xno])
+AM_CONDITIONAL([ENABLE_VALGRIND], [test "x$enable_valgrind" = xyes])
 AM_CONDITIONAL([ENABLE_ASAN], [test "x$enable_asan" = xyes])
 AM_CONDITIONAL([ENABLE_UBSAN], [test "x$enable_ubsan" = xyes])
 AM_CONDITIONAL([ENABLE_GCOV], [test "x$enable_gcov" = xyes])
@@ -291,7 +285,6 @@ AC_SUBST(onig_LDFLAGS)
 
 AM_CONDITIONAL([BUILD_ONIGURUMA], [test "x$build_oniguruma" = xyes])
 AM_CONDITIONAL([WITH_ONIGURUMA], [test "x$with_oniguruma" != xno])
-AC_SUBST([BUNDLER], ["$bundle_cmd"])
 
 AC_CONFIG_MACRO_DIRS([config/m4 m4])
 AC_CONFIG_HEADERS([src/config.h])

src/builtin.c

@@ -5,24 +5,12 @@
 # define _XPG6
 # define __EXTENSIONS__
 #endif
+#ifdef __OpenBSD__
+# define _BSD_SOURCE
+#endif
 #include <sys/time.h>
 #include <stdlib.h>
 #include <stddef.h>
-#ifdef HAVE_ALLOCA_H
-# include <alloca.h>
-#elif !defined alloca
-# ifdef __GNUC__
-#  define alloca __builtin_alloca
-# elif defined _MSC_VER
-#  include <malloc.h>
-#  define alloca _alloca
-# elif !defined HAVE_ALLOCA
-#  ifdef  __cplusplus
-extern "C"
-#  endif
-void *alloca (size_t);
-# endif
-#endif
 #include <assert.h>
 #include <ctype.h>
 #include <limits.h>
@@ -340,21 +328,10 @@ jv binop_multiply(jv a, jv b) {
       str = b;
       num = a;
     }
-    jv res;
     double d = jv_number_value(num);
-    if (d < 0 || isnan(d)) {
-      res = jv_null();
-    } else {
-      int n = d;
-      size_t alen = jv_string_length_bytes(jv_copy(str));
-      res = jv_string_empty(alen * n);
-      for (; n > 0; n--) {
-        res = jv_string_append_buf(res, jv_string_value(str), alen);
-      }
-    }
-    jv_free(str);
     jv_free(num);
-    return res;
+    return jv_string_repeat(str,
+        d < 0 || isnan(d) ? -1 : d > INT_MAX ? INT_MAX : (int)d);
   } else if (ak == JV_KIND_OBJECT && bk == JV_KIND_OBJECT) {
     return jv_object_merge_recursive(a, b);
   } else {
@@ -494,7 +471,7 @@ static jv f_length(jq_state *jq, jv input) {
   } else if (jv_get_kind(input) == JV_KIND_STRING) {
     return jv_number(jv_string_length_codepoints(input));
   } else if (jv_get_kind(input) == JV_KIND_NUMBER) {
-    jv r = jv_number(fabs(jv_number_value(input)));
+    jv r = jv_number_abs(input);
     jv_free(input);
     return r;
   } else if (jv_get_kind(input) == JV_KIND_NULL) {
@@ -757,9 +734,12 @@ static jv f_format(jq_state *jq, jv input, jv fmt) {
     input = f_tostring(jq, input);
     const unsigned char* data = (const unsigned char*)jv_string_value(input);
     int len = jv_string_length_bytes(jv_copy(input));
+    if (len == 0) {
+      jv_free(input);
+      return jv_string("");
+    }
     size_t decoded_len = (3 * (size_t)len) / 4; // 3 usable bytes for every 4 bytes of input
     char *result = jv_mem_calloc(decoded_len, sizeof(char));
-    memset(result, 0, decoded_len * sizeof(char));
     uint32_t ri = 0;
     int input_bytes_read=0;
     uint32_t code = 0;
@@ -879,6 +859,24 @@ static jv f_group_by_impl(jq_state *jq, jv input, jv keys) {
   }
 }
 
+static jv f_unique(jq_state *jq, jv input) {
+  if (jv_get_kind(input) == JV_KIND_ARRAY) {
+    return jv_unique(input, jv_copy(input));
+  } else {
+    return type_error(input, "cannot be sorted, as it is not an array");
+  }
+}
+
+static jv f_unique_by_impl(jq_state *jq, jv input, jv keys) {
+  if (jv_get_kind(input) == JV_KIND_ARRAY &&
+      jv_get_kind(keys) == JV_KIND_ARRAY &&
+      jv_array_length(jv_copy(input)) == jv_array_length(jv_copy(keys))) {
+    return jv_unique(input, keys);
+  } else {
+    return type_error2(input, keys, "cannot be sorted, as they are not both arrays");
+  }
+}
+
 #ifdef HAVE_LIBONIG
 static int f_match_name_iter(const UChar* name, const UChar *name_end, int ngroups,
     int *groups, regex_t *reg, void *arg) {
@@ -1758,16 +1756,16 @@ static jv f_strftime(jq_state *jq, jv a, jv b) {
     return ret_error(b, jv_string("strftime/1 requires parsed datetime inputs"));
 
   const char *fmt = jv_string_value(b);
-  size_t alloced = strlen(fmt) + 100;
-  char *buf = alloca(alloced);
+  size_t max_size = strlen(fmt) + 100;
+  char *buf = jv_mem_alloc(max_size);
 #ifdef __APPLE__
   /* Apple Libc (as of version 1669.40.2) contains a bug which causes it to
    * ignore the `tm.tm_gmtoff` in favor of the global timezone. To print the
    * proper timezone offset we temporarily switch the TZ to UTC. */
   char *tz = (tz = getenv("TZ")) != NULL ? strdup(tz) : NULL;
   setenv("TZ", "UTC", 1);
 #endif
-  size_t n = strftime(buf, alloced, fmt, &tm);
+  size_t n = strftime(buf, max_size, fmt, &tm);
 #ifdef __APPLE__
   if (tz) {
     setenv("TZ", tz, 1);
@@ -1778,9 +1776,13 @@ static jv f_strftime(jq_state *jq, jv a, jv b) {
 #endif
   jv_free(b);
   /* POSIX doesn't provide errno values for strftime() failures; weird */
-  if (n == 0 || n > alloced)
+  if ((n == 0 && *fmt) || n > max_size) {
+    free(buf);
     return jv_invalid_with_msg(jv_string("strftime/1: unknown system failure"));
-  return jv_string(buf);
+  }
+  jv ret = jv_string_sized(buf, n);
+  free(buf);
+  return ret;
 }
 #else
 static jv f_strftime(jq_state *jq, jv a, jv b) {
@@ -1803,14 +1805,18 @@ static jv f_strflocaltime(jq_state *jq, jv a, jv b) {
   if (!jv2tm(a, &tm, 1))
     return ret_error(b, jv_string("strflocaltime/1 requires parsed datetime inputs"));
   const char *fmt = jv_string_value(b);
-  size_t alloced = strlen(fmt) + 100;
-  char *buf = alloca(alloced);
-  size_t n = strftime(buf, alloced, fmt, &tm);
+  size_t max_size = strlen(fmt) + 100;
+  char *buf = jv_mem_alloc(max_size);
+  size_t n = strftime(buf, max_size, fmt, &tm);
   jv_free(b);
   /* POSIX doesn't provide errno values for strftime() failures; weird */
-  if (n == 0 || n > alloced)
+  if ((n == 0 && *fmt) || n > max_size) {
+    free(buf);
     return jv_invalid_with_msg(jv_string("strflocaltime/1: unknown system failure"));
-  return jv_string(buf);
+  }
+  jv ret = jv_string_sized(buf, n);
+  free(buf);
+  return ret;
 }
 #else
 static jv f_strflocaltime(jq_state *jq, jv a, jv b) {
@@ -1912,6 +1918,8 @@ BINOPS
   CFUNC(f_sort, "sort", 1),
   CFUNC(f_sort_by_impl, "_sort_by_impl", 2),
   CFUNC(f_group_by_impl, "_group_by_impl", 2),
+  CFUNC(f_unique, "unique", 1),
+  CFUNC(f_unique_by_impl, "_unique_by_impl", 2),
   CFUNC(f_bsearch, "bsearch", 2),
   CFUNC(f_min, "min", 1),
   CFUNC(f_max, "max", 1),

src/builtin.jq

@@ -4,8 +4,7 @@ def map(f): [.[] | f];
 def select(f): if f then . else empty end;
 def sort_by(f): _sort_by_impl(map([f]));
 def group_by(f): _group_by_impl(map([f]));
-def unique: group_by(.) | map(.[0]);
-def unique_by(f): group_by(f) | map(.[0]);
+def unique_by(f): _unique_by_impl(map([f]));
 def max_by(f): _max_by_impl(map([f]));
 def min_by(f): _min_by_impl(map([f]));
 def add(f): reduce f as $x (null; . + $x);

src/compile.c

@@ -1,13 +1,11 @@
 #include <assert.h>
-#include <math.h>
 #include <string.h>
 #include <stdlib.h>
 #include <unistd.h>
 #include "compile.h"
 #include "bytecode.h"
 #include "locfile.h"
 #include "jv_alloc.h"
-#include "linker.h"
 
 /*
   The intermediate representation for jq filters is as a sequence of
@@ -1374,7 +1372,7 @@ int block_compile(block b, struct bytecode** out, struct locfile* lf, jv args) {
   bc->globals = jv_mem_alloc(sizeof(struct symbol_table));
   int ncfunc = count_cfunctions(b);
   bc->globals->ncfunctions = 0;
-  bc->globals->cfunctions = jv_mem_calloc(ncfunc, sizeof(struct cfunction));
+  bc->globals->cfunctions = jv_mem_calloc(ncfunc ? ncfunc : 1, sizeof(struct cfunction));
   bc->globals->cfunc_names = jv_array();
   bc->debuginfo = jv_object_set(jv_object(), jv_string("name"), jv_null());
   jv env = jv_invalid();