Add support for non-byte-aligned bitstrings

Signed-off-by: Paul Guyot <pguyot@kallisys.net>

Author: pguyot

.github/workflows/build-and-test-macos.yaml

@@ -259,6 +259,12 @@ jobs:
       run: |
         ./tests/test-term
 
+    - name: "Test: test-bitstring"
+      timeout-minutes: 10
+      working-directory: build
+      run: |
+        ./tests/test-bitstring
+
     - name: "Test: test_etest.avm"
       timeout-minutes: 5
       working-directory: build

.github/workflows/build-and-test-on-freebsd.yaml

@@ -164,6 +164,13 @@ jobs:
         cd build
         ./tests/test-term
 
+    - name: "Test: test-bitstring"
+      shell: freebsd {0}
+      run: |
+        cd $GITHUB_WORKSPACE;
+        cd build
+        ./tests/test-bitstring
+
     - name: "Test: test_etest.avm"
       shell: freebsd {0}
       run: |

.github/workflows/build-and-test.yaml

@@ -808,6 +808,19 @@ jobs:
         ulimit -c unlimited
         ./tests/test-erlang -s prime_smp
 
+    - name: "Test: test-bitstring with valgrind"
+      if: matrix.library-arch == ''
+      working-directory: build
+      run: |
+        ulimit -c unlimited
+        valgrind --error-exitcode=1 ./tests/test-bitstring
+
+    - name: "Test: test-bitstring"
+      working-directory: build
+      run: |
+        ulimit -c unlimited
+        ./tests/test-bitstring
+
     - name: "Test: test-enif with valgrind"
       if: matrix.library-arch == ''
       working-directory: build

CHANGELOG.md

@@ -34,6 +34,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added `console:print_err/1` to write to standard error
 - Added `erlang:term_to_binary/2`, `erlang:is_builtin/3` and `erlang:bitstring_to_list/1`
 - Added `lists:mapfoldr/3`
+- Added support for non-byte-aligned bitstrings
 
 ### Changed
 - Updated network type db() to dbm() to reflect the actual representation of the type

code-queries/allocations-exceeding-ensure-free.ql

@@ -950,6 +950,14 @@ predicate consumesContextBudget(FunctionCall efCall, FunctionCall consumer) {
         ensureFreeContextVar(efCall)
 }
 
+/**
+ * Holds if `f` tears down a context (e.g. `context_destroy`): it frees the
+ * context rather than re-establishing a usable heap budget on it.
+ */
+predicate isContextTeardown(Function f) {
+    f.hasName("context_destroy")
+}
+
 /**
  * Holds if `efCall` is a redundant reserving ensure_free: no allocating call
  * uses its budget, and `supersedingCall` is a subsequent call that resets
@@ -959,6 +967,8 @@ predicate consumesContextBudget(FunctionCall efCall, FunctionCall consumer) {
 pragma[nomagic]
 predicate isRedundantEnsureFree(FunctionCall efCall, FunctionCall supersedingCall) {
     isReservingEnsureFreeCall(efCall) and
+    // Test code calls ensure_free for its GC side effects
+    not efCall.getFile().getRelativePath().matches("tests/%") and
     // No allocating call uses this ensure_free's budget
     not exists(FunctionCall a | allocToBudget(a, efCall)) and
     // ...and no pointer-bumping consumer (memory_copy_term_tree) uses it either
@@ -986,8 +996,10 @@ predicate isRedundantEnsureFree(FunctionCall efCall, FunctionCall supersedingCal
             // A function that internally calls ensure_free on the caller's
             // context (e.g., enif_make_resource), passed that same context as an
             // argument. Uses the ensure_free-only notion: own-heap setup does not
-            // reset this context's budget.
+            // reset this context's budget. Context teardown (context_destroy) is
+            // excluded: it frees the context rather than re-establishing a budget.
             not isEnsureFreeCall(supersedingCall) and
+            not isContextTeardown(supersedingCall.getTarget()) and
             transitivelyCallsEnsureFreeOnly(supersedingCall.getTarget()) and
             supersedingCall.getAnArgument().(VariableAccess).getTarget() = ctxVar
         ) and

doc/src/apidocs/libatomvm/functions.rst

@@ -242,7 +242,6 @@ Functions
 .. doxygenfunction:: term_binary_heap_size
 .. doxygenfunction:: term_binary_size_is_heap_binary
 .. doxygenfunction:: term_boxed_size
-.. doxygenfunction:: term_bs_insert_binary
 .. doxygenfunction:: term_compare
 .. doxygenfunction:: term_create_empty_binary
 .. doxygenfunction:: term_create_uninitialized_binary

libs/estdlib/src/erlang.erl

@@ -155,6 +155,7 @@
     hd/1,
     is_atom/1,
     is_binary/1,
+    is_bitstring/1,
     is_boolean/1,
     is_float/1,
     is_function/1,
@@ -933,15 +934,16 @@ binary_to_list(_Binary) ->
 
 %%-----------------------------------------------------------------------------
 %% @param   Bitstring   Bitstring to convert to list
-%% @returns a list of bytes from the bitstring
+%% @returns a list of bytes, with a final element holding the trailing bits if
+%%          `Bitstring' is not a whole number of bytes
 %% @doc     Convert a bitstring to a list of bytes.
 %%
-%% Unlike Erlang/OTP, AtomVM only supports byte-aligned bitstrings (binaries),
-%% so the returned list never has a trailing bitstring and this function
-%% behaves like `binary_to_list/1'.
+%%          If the number of bits in `Bitstring' is not divisible by `8', the
+%%          last element of the list is a bitstring containing the trailing
+%%          `1..7' bits, e.g. `bitstring_to_list(<<1:1>>)' returns `[<<1:1>>]'.
 %% @end
 %%-----------------------------------------------------------------------------
--spec bitstring_to_list(Bitstring :: bitstring()) -> [byte()].
+-spec bitstring_to_list(Bitstring :: bitstring()) -> [byte() | bitstring()].
 bitstring_to_list(_Bitstring) ->
     erlang:nif_error(undefined).
 
@@ -1973,6 +1975,19 @@ is_atom(_Term) ->
 is_binary(_Term) ->
     erlang:nif_error(undefined).
 
+%%-----------------------------------------------------------------------------
+%% @param   Term  the term to test
+%% @returns `true' if `Term' is a bitstring; `false', otherwise.
+%% @doc     Return `true' if `Term' is a bitstring (including a binary);
+%%          `false', otherwise.
+%%
+%% This function may be used in a guard expression.
+%% @end
+%%-----------------------------------------------------------------------------
+-spec is_bitstring(Term :: term()) -> boolean().
+is_bitstring(_Term) ->
+    erlang:nif_error(undefined).
+
 %%-----------------------------------------------------------------------------
 %% @param   Term  the term to test
 %% @returns `true' if `Term' is a boolean (`true' or `false'); `false', otherwise.

libs/jit/src/default_atoms.hrl

@@ -37,6 +37,7 @@
 -define(ALL_ATOM_INDEX, 15).
 -define(LOWERCASE_EXIT_ATOM_INDEX, 16).
 -define(BADRECORD_ATOM_INDEX, 17).
+-define(SYSTEM_LIMIT_ATOM_INDEX, 20).
 
 -define(FALSE_ATOM, ((?FALSE_ATOM_INDEX bsl ?TERM_IMMED2_TAG_SIZE) bor ?TERM_IMMED2_ATOM)).
 -define(TRUE_ATOM, ((?TRUE_ATOM_INDEX bsl ?TERM_IMMED2_TAG_SIZE) bor ?TERM_IMMED2_ATOM)).
@@ -68,6 +69,9 @@
     ((?LOWERCASE_EXIT_ATOM_INDEX bsl ?TERM_IMMED2_TAG_SIZE) bor ?TERM_IMMED2_ATOM)
 ).
 -define(BADRECORD_ATOM, ((?BADRECORD_ATOM_INDEX bsl ?TERM_IMMED2_TAG_SIZE) bor ?TERM_IMMED2_ATOM)).
+-define(SYSTEM_LIMIT_ATOM,
+    ((?SYSTEM_LIMIT_ATOM_INDEX bsl ?TERM_IMMED2_TAG_SIZE) bor ?TERM_IMMED2_ATOM)
+).
 
 -define(DEFAULT_ATOMS, #{
     false => ?FALSE_ATOM,