From b7188f6a29d670b8442501a291c3980db1a90ce5 Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Mon, 2 Jun 2025 14:27:36 +0800 Subject: [PATCH 1/2] support meta shared serialization for xlang in java --- .../src/main/java/org/apache/fory/Fory.java | 24 +++++++--- .../fory/builder/MetaSharedCodecBuilder.java | 6 ++- .../fory/builder/ObjectCodecBuilder.java | 4 +- .../org/apache/fory/config/ForyBuilder.java | 3 ++ .../java/org/apache/fory/meta/ClassDef.java | 30 ++++++++----- .../fory/serializer/MetaSharedSerializer.java | 20 +++++++-- .../NonexistentClassSerializers.java | 6 ++- .../fory/serializer/ObjectSerializer.java | 2 +- .../fory/serializer/SerializationUtils.java | 2 +- .../apache/fory/serializer/Serializer.java | 3 +- .../fory/xlang/MetaSharedXlangTest.java | 45 +++++++++++++++++++ 11 files changed, 119 insertions(+), 26 deletions(-) create mode 100644 java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java diff --git a/java/fory-core/src/main/java/org/apache/fory/Fory.java b/java/fory-core/src/main/java/org/apache/fory/Fory.java index c26c01bb83..9b097b1636 100644 --- a/java/fory-core/src/main/java/org/apache/fory/Fory.java +++ b/java/fory-core/src/main/java/org/apache/fory/Fory.java @@ -326,8 +326,7 @@ public MemoryBuffer serialize(MemoryBuffer buffer, Object obj, BufferCallback ca if (language == Language.JAVA) { write(buffer, obj); } else { - buffer.writeByte((byte) Language.JAVA.ordinal()); - xwriteRef(buffer, obj); + xwrite(buffer, obj); } return buffer; } catch (StackOverflowError t) { @@ -412,6 +411,21 @@ private void write(MemoryBuffer buffer, Object obj) { } } + private void xwrite(MemoryBuffer buffer, Object obj) { + buffer.writeByte((byte) Language.JAVA.ordinal()); + int startOffset = buffer.writerIndex(); + boolean shareMeta = config.isMetaShareEnabled(); + if (shareMeta) { + buffer.writeInt32(-1); // preserve 4-byte for meta start offsets. + } + xwriteRef(buffer, obj); + MetaContext metaContext = serializationContext.getMetaContext(); + if (shareMeta && metaContext != null && !metaContext.writingClassDefs.isEmpty()) { + buffer.putInt32(startOffset, buffer.writerIndex() - startOffset - 4); + classResolver.writeClassDefs(buffer); + } + } + /** Serialize a nullable referencable object to buffer. */ public void writeRef(MemoryBuffer buffer, Object obj) { if (!refResolver.writeRefOrNull(buffer, obj)) { @@ -861,13 +875,13 @@ public Object deserialize(MemoryBuffer buffer, Iterable outOfBandB "outOfBandBuffers should be null when the serialized stream is " + "produced with bufferCallback null."); } + if (shareMeta) { + readClassDefs(buffer); + } Object obj; if (isTargetXLang) { obj = xreadRef(buffer); } else { - if (shareMeta) { - readClassDefs(buffer); - } obj = readRef(buffer); } return obj; diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java index f9308ee535..099b0081dd 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/MetaSharedCodecBuilder.java @@ -76,7 +76,11 @@ public MetaSharedCodecBuilder(TypeRef beanType, Fory fory, ClassDef classDef) this.classDef = classDef; Collection descriptors = fory( - f -> MetaSharedSerializer.consolidateFields(f.getClassResolver(), beanClass, classDef)); + f -> + MetaSharedSerializer.consolidateFields( + f.isCrossLanguage() ? f.getXtypeResolver() : f.getClassResolver(), + beanClass, + classDef)); DescriptorGrouper grouper = fory.getClassResolver().createDescriptorGrouper(descriptors, false); objectCodecOptimizer = new ObjectCodecOptimizer(beanClass, grouper, !fory.isBasicTypesRefIgnored(), ctx); diff --git a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java index ed07302e4e..b4e525d7f6 100644 --- a/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java @@ -88,13 +88,15 @@ public ObjectCodecBuilder(Class beanClass, Fory fory) { super(TypeRef.of(beanClass), fory, Generated.GeneratedObjectSerializer.class); Collection descriptors; boolean shareMeta = fory.getConfig().isMetaShareEnabled(); + boolean xlang = fory.isCrossLanguage(); if (shareMeta) { descriptors = fory( f -> f.getClassResolver() .getClassDef(beanClass, true) - .getDescriptors(f.getClassResolver(), beanClass)); + .getDescriptors( + xlang ? f.getXtypeResolver() : f.getClassResolver(), beanClass)); } else { descriptors = fory.getClassResolver().getFieldDescriptors(beanClass, true); } diff --git a/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java b/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java index 7edcf02108..cd3955a1ce 100644 --- a/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java +++ b/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java @@ -95,6 +95,9 @@ public ForyBuilder() {} */ public ForyBuilder withLanguage(Language language) { this.language = language; + if (language != Language.JAVA) { + codeGenEnabled = false; + } return this; } diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/ClassDef.java b/java/fory-core/src/main/java/org/apache/fory/meta/ClassDef.java index cef444b63a..26d7a3f243 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/ClassDef.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/ClassDef.java @@ -239,9 +239,10 @@ public static ClassDef readClassDef(Fory fory, MemoryBuffer buffer, long header) * * @param cls class load in current process. */ - public List getDescriptors(ClassResolver resolver, Class cls) { + public List getDescriptors(TypeResolver resolver, Class cls) { if (descriptors == null) { - SortedMap allDescriptorsMap = resolver.getAllDescriptorsMap(cls, true); + SortedMap allDescriptorsMap = + resolver.getFory().getClassResolver().getAllDescriptorsMap(cls, true); Map descriptorsMap = new HashMap<>(); for (Map.Entry e : allDescriptorsMap.entrySet()) { if (descriptorsMap.put( @@ -333,9 +334,9 @@ public FieldType getFieldType() { * null. Don't invoke this method if class does have fieldName field. In such case, * reflection should be used to get the descriptor. */ - Descriptor toDescriptor(ClassResolver classResolver, Descriptor descriptor) { + Descriptor toDescriptor(TypeResolver resolver, Descriptor descriptor) { TypeRef declared = descriptor != null ? descriptor.getTypeRef() : null; - TypeRef typeRef = fieldType.toTypeToken(classResolver, declared); + TypeRef typeRef = fieldType.toTypeToken(resolver, declared); // This field doesn't exist in peer class, so any legal modifier will be OK. int stubModifiers = ReflectionUtils.getField(getClass(), "fieldName").getModifiers(); return new Descriptor(typeRef, fieldName, stubModifiers, definedClass); @@ -407,7 +408,7 @@ public boolean nullable() { * * @see FinalObjectTypeStub */ - public abstract TypeRef toTypeToken(ClassResolver classResolver, TypeRef declared); + public abstract TypeRef toTypeToken(TypeResolver classResolver, TypeRef declared); @Override public boolean equals(Object o) { @@ -584,8 +585,13 @@ public short getClassId() { } @Override - public TypeRef toTypeToken(ClassResolver classResolver, TypeRef declared) { - Class cls = classResolver.getRegisteredClass(classId); + public TypeRef toTypeToken(TypeResolver resolver, TypeRef declared) { + Class cls; + if (resolver instanceof XtypeResolver) { + cls = ((XtypeResolver) resolver).getXtypeInfo(classId).getCls(); + } else { + cls = ((ClassResolver) resolver).getRegisteredClass(classId); + } if (cls == null) { LOG.warn("Class {} not registered, take it as Struct type for deserialization.", classId); cls = NonexistentClass.NonexistentMetaShared.class; @@ -652,7 +658,7 @@ public FieldType getElementType() { } @Override - public TypeRef toTypeToken(ClassResolver classResolver, TypeRef declared) { + public TypeRef toTypeToken(TypeResolver classResolver, TypeRef declared) { // TODO support preserve element TypeExtMeta TypeRef> collectionTypeRef = collectionOf( @@ -749,7 +755,7 @@ public FieldType getValueType() { } @Override - public TypeRef toTypeToken(ClassResolver classResolver, TypeRef declared) { + public TypeRef toTypeToken(TypeResolver classResolver, TypeRef declared) { // TODO support preserve element TypeExtMeta, it will be lost when building other TypeRef return mapOf( keyType.toTypeToken(classResolver, declared), @@ -798,7 +804,7 @@ private EnumFieldType(boolean nullable, int xtypeId) { } @Override - public TypeRef toTypeToken(ClassResolver classResolver, TypeRef declared) { + public TypeRef toTypeToken(TypeResolver classResolver, TypeRef declared) { return TypeRef.of(NonexistentClass.NonexistentEnum.class); } } @@ -825,7 +831,7 @@ public ArrayFieldType( } @Override - public TypeRef toTypeToken(ClassResolver classResolver, TypeRef declared) { + public TypeRef toTypeToken(TypeResolver classResolver, TypeRef declared) { TypeRef componentTypeRef = componentType.toTypeToken(classResolver, declared); Class componentRawType = componentTypeRef.getRawType(); if (NonexistentClass.class.isAssignableFrom(componentRawType)) { @@ -893,7 +899,7 @@ public ObjectFieldType(int xtypeId, boolean isFinal, boolean nullable, boolean t } @Override - public TypeRef toTypeToken(ClassResolver classResolver, TypeRef declared) { + public TypeRef toTypeToken(TypeResolver classResolver, TypeRef declared) { return isMonomorphic() ? TypeRef.of(FinalObjectTypeStub.class, new TypeExtMeta(nullable, trackingRef)) : TypeRef.of(Object.class, new TypeExtMeta(nullable, trackingRef)); diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java index 8ac9c022b0..4d49e25783 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/MetaSharedSerializer.java @@ -36,6 +36,7 @@ import org.apache.fory.resolver.ClassInfoHolder; import org.apache.fory.resolver.ClassResolver; import org.apache.fory.resolver.RefResolver; +import org.apache.fory.resolver.TypeResolver; import org.apache.fory.type.Descriptor; import org.apache.fory.type.DescriptorGrouper; import org.apache.fory.type.Generics; @@ -84,7 +85,10 @@ public MetaSharedSerializer(Fory fory, Class type, ClassDef classDef) { "Class version check should be disabled when compatible mode is enabled."); Preconditions.checkArgument( fory.getConfig().isMetaShareEnabled(), "Meta share must be enabled."); - Collection descriptors = consolidateFields(this.classResolver, type, classDef); + boolean xlang = fory.isCrossLanguage(); + Collection descriptors = + consolidateFields( + xlang ? fory.getXtypeResolver() : fory.getClassResolver(), type, classDef); DescriptorGrouper descriptorGrouper = classResolver.createDescriptorGrouper(descriptors, false); // d.getField() may be null if not exists in this class when meta share enabled. Tuple3< @@ -118,6 +122,11 @@ public void write(MemoryBuffer buffer, T value) { serializer.write(buffer, value); } + @Override + public void xwrite(MemoryBuffer buffer, T value) { + write(buffer, value); + } + @Override public T read(MemoryBuffer buffer) { if (isRecord) { @@ -192,6 +201,11 @@ public T read(MemoryBuffer buffer) { return obj; } + @Override + public T xread(MemoryBuffer buffer) { + return read(buffer); + } + private void readFields(MemoryBuffer buffer, Object[] fields) { int counter = 0; Fory fory = this.fory; @@ -280,7 +294,7 @@ static boolean skipPrimitiveFieldValueFailed(Fory fory, short classId, MemoryBuf } public static Collection consolidateFields( - ClassResolver classResolver, Class cls, ClassDef classDef) { - return classDef.getDescriptors(classResolver, cls); + TypeResolver resolver, Class cls, ClassDef classDef) { + return classDef.getDescriptors(resolver, cls); } } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java index 4c48e6c3b5..70d4d1cada 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/NonexistentClassSerializers.java @@ -19,6 +19,8 @@ package org.apache.fory.serializer; +import static org.apache.fory.serializer.SerializationUtils.getTypeResolver; + import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -36,6 +38,7 @@ import org.apache.fory.resolver.MetaContext; import org.apache.fory.resolver.MetaStringResolver; import org.apache.fory.resolver.RefResolver; +import org.apache.fory.resolver.TypeResolver; import org.apache.fory.serializer.NonexistentClass.NonexistentEnum; import org.apache.fory.type.Descriptor; import org.apache.fory.type.DescriptorGrouper; @@ -155,11 +158,12 @@ public void write(MemoryBuffer buffer, Object v) { private ClassFieldsInfo getClassFieldsInfo(ClassDef classDef) { ClassFieldsInfo fieldsInfo = fieldsInfoMap.get(classDef.getId()); + TypeResolver resolver = getTypeResolver(fory); if (fieldsInfo == null) { // Use `NonexistentSkipClass` since it doesn't have any field. Collection descriptors = MetaSharedSerializer.consolidateFields( - fory.getClassResolver(), NonexistentClass.NonexistentSkip.class, classDef); + resolver, NonexistentClass.NonexistentSkip.class, classDef); DescriptorGrouper descriptorGrouper = fory.getClassResolver().createDescriptorGrouper(descriptors, false); Tuple3< diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java index c3adf22464..741491a0c0 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectSerializer.java @@ -96,7 +96,7 @@ public ObjectSerializer(Fory fory, Class cls, boolean resolveParent) { boolean shareMeta = fory.getConfig().isMetaShareEnabled(); if (shareMeta) { ClassDef classDef = classResolver.getClassDef(cls, resolveParent); - descriptors = classDef.getDescriptors(classResolver, cls); + descriptors = classDef.getDescriptors(typeResolver, cls); } else { descriptors = fory.getClassResolver().getFieldDescriptors(cls, resolveParent); } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationUtils.java b/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationUtils.java index c88910db67..51760f7350 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationUtils.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/SerializationUtils.java @@ -26,7 +26,7 @@ @Internal class SerializationUtils { - static TypeResolver getTypeResolver(Fory fory) { + public static TypeResolver getTypeResolver(Fory fory) { return fory.isCrossLanguage() ? fory.getXtypeResolver() : fory.getClassResolver(); } diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializer.java index ef10c46026..076d5ae65a 100644 --- a/java/fory-core/src/main/java/org/apache/fory/serializer/Serializer.java +++ b/java/fory-core/src/main/java/org/apache/fory/serializer/Serializer.java @@ -67,7 +67,8 @@ public void xwrite(MemoryBuffer buffer, T value) { } public T xread(MemoryBuffer buffer) { - throw new UnsupportedOperationException(); + throw new UnsupportedOperationException( + getClass() + " doesn't support xlang serialization for " + type); } public Serializer(Fory fory, Class type) { diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java new file mode 100644 index 0000000000..020da26a9c --- /dev/null +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fory.xlang; + +import org.apache.fory.CrossLanguageTest.Bar; +import org.apache.fory.CrossLanguageTest.Foo; +import org.apache.fory.Fory; +import org.apache.fory.ForyTestBase; +import org.apache.fory.config.CompatibleMode; +import org.apache.fory.config.Language; +import org.testng.annotations.Test; + +public class MetaSharedXlangTest extends ForyTestBase { + + @Test + public void testMetaShared() { + Fory fory = + Fory.builder() + .withLanguage(Language.XLANG) + .withCodegen(false) + .withCompatibleMode(CompatibleMode.COMPATIBLE) + .build(); + fory.register(Foo.class, "example.foo"); + fory.register(Bar.class, "example.bar"); + serDeCheck(fory, Bar.create()); + serDeCheck(fory, Foo.create()); + } +} From 56d40e1b5c5a6e1f0a67af20678384154612ec7a Mon Sep 17 00:00:00 2001 From: chaokunyang Date: Mon, 2 Jun 2025 15:21:27 +0800 Subject: [PATCH 2/2] fix array field serialization --- .../java/org/apache/fory/meta/ClassDef.java | 19 +++++++--- .../fory/xlang/MetaSharedXlangTest.java | 36 ++++++++++++++++++- 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/ClassDef.java b/java/fory-core/src/main/java/org/apache/fory/meta/ClassDef.java index 26d7a3f243..f5c9e41fa1 100644 --- a/java/fory-core/src/main/java/org/apache/fory/meta/ClassDef.java +++ b/java/fory-core/src/main/java/org/apache/fory/meta/ClassDef.java @@ -337,6 +337,13 @@ public FieldType getFieldType() { Descriptor toDescriptor(TypeResolver resolver, Descriptor descriptor) { TypeRef declared = descriptor != null ? descriptor.getTypeRef() : null; TypeRef typeRef = fieldType.toTypeToken(resolver, declared); + if (descriptor != null) { + if (typeRef.equals(declared)) { + return descriptor; + } else { + descriptor.copyWithTypeName(typeRef.getType().getTypeName()); + } + } // This field doesn't exist in peer class, so any legal modifier will be OK. int stubModifiers = ReflectionUtils.getField(getClass(), "fieldName").getModifiers(); return new Descriptor(typeRef, fieldName, stubModifiers, definedClass); @@ -556,10 +563,6 @@ public static FieldType xread( ClassInfo classInfo = resolver.getXtypeInfo(xtypeId); Preconditions.checkNotNull(classInfo); Class cls = classInfo.getCls(); - if (Types.isPrimitiveArray(xtypeId)) { - FieldType type = buildFieldType(resolver, resolver.buildGenericType(cls)); - return new ArrayFieldType(xtypeId, true, nullable, trackingRef, type, 1); - } return new RegisteredFieldType( resolver.isMonomorphic(cls), nullable, trackingRef, xtypeId); } else { @@ -589,6 +592,14 @@ public TypeRef toTypeToken(TypeResolver resolver, TypeRef declared) { Class cls; if (resolver instanceof XtypeResolver) { cls = ((XtypeResolver) resolver).getXtypeInfo(classId).getCls(); + if (Types.isPrimitiveType(classId)) { + if (declared.isPrimitive() && !nullable) { + cls = TypeUtils.unwrap(cls); + } + if (nullable && !declared.isPrimitive()) { + cls = TypeUtils.wrap(cls); + } + } } else { cls = ((ClassResolver) resolver).getRegisteredClass(classId); } diff --git a/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java index 020da26a9c..0528c8342f 100644 --- a/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java +++ b/java/fory-core/src/test/java/org/apache/fory/xlang/MetaSharedXlangTest.java @@ -19,18 +19,20 @@ package org.apache.fory.xlang; +import lombok.Data; import org.apache.fory.CrossLanguageTest.Bar; import org.apache.fory.CrossLanguageTest.Foo; import org.apache.fory.Fory; import org.apache.fory.ForyTestBase; import org.apache.fory.config.CompatibleMode; import org.apache.fory.config.Language; +import org.apache.fory.test.bean.BeanB; import org.testng.annotations.Test; public class MetaSharedXlangTest extends ForyTestBase { @Test - public void testMetaShared() { + public void testMetaSharedBasic() { Fory fory = Fory.builder() .withLanguage(Language.XLANG) @@ -42,4 +44,36 @@ public void testMetaShared() { serDeCheck(fory, Bar.create()); serDeCheck(fory, Foo.create()); } + + @Test + public void testMetaSharedComplex1() { + Fory fory = + Fory.builder() + .withLanguage(Language.XLANG) + .withCodegen(false) + .withCompatibleMode(CompatibleMode.COMPATIBLE) + .build(); + fory.register(BeanB.class, "example.b"); + serDeCheck(fory, BeanB.createBeanB(2)); + } + + @Data + static class MDArrayFieldStruct { + int[][] arr; + } + + // @Test + public void testMDArrayField() { + Fory fory = + Fory.builder() + .withLanguage(Language.XLANG) + .withCodegen(false) + .withCompatibleMode(CompatibleMode.COMPATIBLE) + .build(); + // TODO support multi-dimensional array serialization + fory.register(MDArrayFieldStruct.class, "example.a"); + MDArrayFieldStruct s = new MDArrayFieldStruct(); + s.arr = new int[][] {{1, 2}, {3, 4}}; + serDeCheck(fory, s); + } }