Add VarZeroLengthlessSlice, use in MultiFieldsULE to optimize

CHANGELOG.md

@@ -37,6 +37,8 @@
     - `zerofrom`
     - `zerotrie`
     - `zerovec`
+      - This release has multiple changes that affect the bit representation of various types. Do not update to this release if you wish to retain stable data formats.
+        - Optimize `MultiFieldsULE` to not store a length anymore. This breaks data layout for any `#[make_varule]`-using struct with multiple variable-sized fields. (https://github.com/unicode-org/icu4x/pull/5593)
     - `writeable`
 
 

components/experimental/tests/transliterate/data/gen.sh

@@ -1,5 +1,5 @@
 #!/bin/sh
-cargo run -p icu4x-datagen --features experimental_components -- \
+cargo run -p icu4x-datagen --features experimental -- \
 --markers TransliteratorRulesV1Marker \
 --locales full \
 --deduplication none \

components/experimental/tests/transliterate/data/provider.rs

@@ -1,4 +1,4 @@
-include!("baked/macros.rs");
+include!("baked/mod.rs");
 
 pub struct TestingProvider;
 
@@ -17,5 +17,5 @@ const _: () = {
     impl_compatibility_decomposition_supplement_v1_marker!(TestingProvider);
     impl_compatibility_decomposition_tables_v1_marker!(TestingProvider);
     impl_uts46_decomposition_supplement_v1_marker!(TestingProvider);
-    impl_transliterator_rules_v1!(TestingProvider);
+    impl_transliterator_rules_v1_marker!(TestingProvider);
 };

provider/data/experimental/fingerprints.csv

@@ -97741,5 +97741,5 @@ units/essentials@1, zh/long, 423B, 332B, dc8f6b0dd31ee068
 units/essentials@1, zu/long, 476B, 385B, ecb1c4bd6dde6e84
 units/essentials@1, zu/narrow, 372B, 281B, a98ff4ca98d94d42
 units/essentials@1, zu/short, 370B, 279B, 6679b74467716e8c
-units/info@1, <singleton>, 8106B, 8063B, 7de0c4710a635187
+units/info@1, <singleton>, 7486B, 7443B, cbf5defdd14c97d5
 units/trie@1, <singleton>, 1356B, 1335B, c02e6bb603fb4d72

provider/data/locale/fingerprints.csv

@@ -1,4 +1,4 @@
-locale/aliases@2, <singleton>, 8623B, 8171B, 6dffa6ad8d35f77b
+locale/aliases@2, <singleton>, 8547B, 8095B, 3b5578f2a937e797
 locale/exemplarchars/auxiliary@1, <lookup>, 802B, 163 identifiers
 locale/exemplarchars/auxiliary@1, <total>, 42551B, 34828B, 144 unique payloads
 locale/exemplarchars/auxiliary@1, af, 104B, 50B, cb103acefaf75d10

utils/zerovec/derive/src/make_varule.rs

@@ -461,10 +461,11 @@ impl<'a> UnsizedFields<'a> {
 
     // Get the corresponding VarULE type that can store all of these
     fn varule_ty(&self) -> TokenStream2 {
-        if self.fields.len() == 1 {
+        let len = self.fields.len();
+        if len == 1 {
             self.fields[0].kind.varule_ty()
         } else {
-            quote!(zerovec::ule::MultiFieldsULE)
+            quote!(zerovec::ule::MultiFieldsULE::<#len>)
         }
     }
 
@@ -507,7 +508,8 @@ impl<'a> UnsizedFields<'a> {
 
     // Takes all unsized fields on self and encodes them into a byte slice `out`
     fn encode_write(&self, out: TokenStream2) -> TokenStream2 {
-        if self.fields.len() == 1 {
+        let len = self.fields.len();
+        if len == 1 {
             self.fields[0].encode_func(quote!(encode_var_ule_write), quote!(#out))
         } else {
             let mut lengths = vec![];
@@ -523,7 +525,7 @@ impl<'a> UnsizedFields<'a> {
             quote!(
                 let lengths = [#(#lengths),*];
                 // Todo: index type should be settable by attribute
-                let mut multi = zerovec::ule::MultiFieldsULE::<zerovec::vecs::Index32>::new_from_lengths_partially_initialized(&lengths, #out);
+                let mut multi = zerovec::ule::MultiFieldsULE::<#len, zerovec::vecs::Index32>::new_from_lengths_partially_initialized(lengths, #out);
                 unsafe {
                     #(#writers;)*
                 }
@@ -533,15 +535,16 @@ impl<'a> UnsizedFields<'a> {
 
     // Takes all unsized fields on self and returns the length needed for encoding into a byte slice
     fn encode_len(&self) -> TokenStream2 {
-        if self.fields.len() == 1 {
+        let len = self.fields.len();
+        if len == 1 {
             self.fields[0].encode_func(quote!(encode_var_ule_len), quote!())
         } else {
             let mut lengths = vec![];
             for field in self.fields.iter() {
                 lengths.push(field.encode_func(quote!(encode_var_ule_len), quote!()));
             }
             // Todo: index type should be settable by attribute
-            quote!(zerovec::ule::MultiFieldsULE::<zerovec::vecs::Index32>::compute_encoded_len_for(&[#(#lengths),*]))
+            quote!(zerovec::ule::MultiFieldsULE::<#len, zerovec::vecs::Index32>::compute_encoded_len_for([#(#lengths),*]))
         }
     }
 
@@ -597,7 +600,8 @@ impl<'a> UnsizedFields<'a> {
     ///
     /// The code will validate a variable known as `last_field_bytes`
     fn varule_validator(&self) -> Option<TokenStream2> {
-        if self.fields.len() == 1 {
+        let len = self.fields.len();
+        if len == 1 {
             None
         } else {
             let mut validators = vec![];
@@ -607,7 +611,7 @@ impl<'a> UnsizedFields<'a> {
             }
 
             Some(quote!(
-                let multi = zerovec::ule::MultiFieldsULE::parse_byte_slice(last_field_bytes)?;
+                let multi = zerovec::ule::MultiFieldsULE::<#len>::parse_byte_slice(last_field_bytes)?;
                 unsafe {
                     #(#validators)*
                 }