From 8ce5f13ab6f42b300707d82295ed4d4e76e92a2f Mon Sep 17 00:00:00 2001 From: hening Date: Mon, 8 Jul 2024 20:10:20 +0800 Subject: [PATCH 01/18] map general write --- .../collection/AbstractMapSerializer.java | 229 +++++++++++++++++- .../fury/serializer/collection/MapFlags.java | 66 +++++ 2 files changed, 291 insertions(+), 4 deletions(-) create mode 100644 java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapFlags.java diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java index 9e979442da..7fc013d15f 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java @@ -29,6 +29,7 @@ import org.apache.fury.memory.MemoryBuffer; import org.apache.fury.reflect.ReflectionUtils; import org.apache.fury.reflect.TypeRef; +import org.apache.fury.resolver.ClassInfo; import org.apache.fury.resolver.ClassInfoHolder; import org.apache.fury.resolver.ClassResolver; import org.apache.fury.resolver.RefResolver; @@ -40,6 +41,7 @@ /** Serializer for all map-like objects. */ @SuppressWarnings({"unchecked", "rawtypes"}) public abstract class AbstractMapSerializer extends Serializer { + private static final int MAX_CHUNK_SIZE = 128; protected MethodHandle constructor; protected final boolean supportCodegenHook; private Serializer keySerializer; @@ -159,7 +161,7 @@ private void genericJavaWrite(Fury fury, MemoryBuffer buffer, Map map) { Generics generics = fury.getGenerics(); GenericType genericType = generics.nextGenericType(); if (genericType == null) { - generalJavaWrite(fury, buffer, map); + generalJavaWriteV2(fury, buffer, map); } else { GenericType keyGenericType = genericType.getTypeParameter0(); GenericType valueGenericType = genericType.getTypeParameter1(); @@ -169,7 +171,7 @@ private void genericJavaWrite(Fury fury, MemoryBuffer buffer, Map map) { if (genericType.getTypeParametersCount() < 2) { Tuple2 kvGenericType = getKVGenericType(genericType); if (keyGenericType == objType && valueGenericType == objType) { - generalJavaWrite(fury, buffer, map); + generalJavaWriteV2(fury, buffer, map); return; } keyGenericType = kvGenericType.f0; @@ -328,6 +330,174 @@ private void generalJavaWrite(Fury fury, MemoryBuffer buffer, Map map) { } } + private static class Chunk { + + public Chunk(MemoryBuffer memoryBuffer, Fury fury) { + final int writerIndex = memoryBuffer.writerIndex(); + // preserve a byte for header and chunk size + memoryBuffer.writerIndex(writerIndex + 2); + this.startOffset = writerIndex; + this.fury = fury; + } + + public boolean predict(Object key, Object value) { + if (key == null || value == null) { + return true; + } + if (keyClass == null) { + keyClass = key.getClass(); + } + if (valueClass == null) { + valueClass = value.getClass(); + } + return key.getClass() == keyClass + && value.getClass() == valueClass; + } + + private int header = 0; + private int startOffset; + private int chunkSize; + private Class keyClass = null; + private Class valueClass = null; + private final Fury fury; + private boolean writeKeyClassInfo = false; + private boolean writeValueClassInfo = false; + + public void write(Object key, Object value, MemoryBuffer memoryBuffer, ClassInfoHolder keyClassInfoWriteCache, ClassInfoHolder valueClassInfoWriteCache) { + //todo next chunk + if (chunkSize >= MAX_CHUNK_SIZE) { + reset(memoryBuffer); + } + updateHeader(key, value, memoryBuffer, keyClassInfoWriteCache, valueClassInfoWriteCache); + if ((header & MapFlags.TRACKING_KEY_REF) == MapFlags.TRACKING_KEY_REF) { + RefResolver refResolver = fury.getRefResolver(); + if (!refResolver.writeRefOrNull(memoryBuffer, key)) { + keyClassInfoWriteCache.getSerializer().write(memoryBuffer, key); + } + } else { + Serializer serializer = keyClassInfoWriteCache.getSerializer(); + if ((header & MapFlags.KEY_HAS_NULL) != MapFlags.KEY_HAS_NULL) { + serializer.write(memoryBuffer, key); + } else { + if (key == null) { + memoryBuffer.writeByte(Fury.NULL_FLAG); + } else { + memoryBuffer.writeByte(Fury.NOT_NULL_VALUE_FLAG); + serializer.write(memoryBuffer, key); + } + } + } + if ((header & MapFlags.TRACKING_VALUE_REF) == MapFlags.TRACKING_VALUE_REF) { + RefResolver refResolver = fury.getRefResolver(); + if (!refResolver.writeRefOrNull(memoryBuffer, key)) { + valueClassInfoWriteCache.getSerializer().write(memoryBuffer, key); + } + } else { + Serializer serializer = valueClassInfoWriteCache.getSerializer(); + if ((header & MapFlags.VALUE_HAS_NULL) != MapFlags.VALUE_HAS_NULL) { + serializer.write(memoryBuffer, value); + } else { + if (value == null) { + memoryBuffer.writeByte(Fury.NULL_FLAG); + } else { + memoryBuffer.writeByte(Fury.NOT_NULL_VALUE_FLAG); + serializer.write(memoryBuffer, value); + } + } + } + chunkSize++; + } + + public void updateHeader(Object key, Object value, MemoryBuffer memoryBuffer, ClassInfoHolder keyClassInfoWriteCache, ClassInfoHolder valueClassInfoWriteCache) { + //todo 如果key只有null + if (key == null) { + header |= MapFlags.KEY_HAS_NULL; + } else { + ClassResolver classResolver = fury.getClassResolver(); + ClassInfo classInfo = classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache); + if (classInfo.getSerializer().needToWriteRef()) { + header |= MapFlags.TRACKING_KEY_REF; + } + if (!writeKeyClassInfo) { + classResolver.writeClass(memoryBuffer, classInfo); + } + } + if (value == null) { + header |= MapFlags.VALUE_HAS_NULL; + } else { + ClassResolver classResolver = fury.getClassResolver(); + ClassInfo classInfo = classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache); + if (classInfo.getSerializer().needToWriteRef()) { + header |= MapFlags.TRACKING_VALUE_REF; + } + if (!writeValueClassInfo) { + classResolver.writeClass(memoryBuffer, classInfo); + } + } + } + + /** + * update chunk size and header, if chunk size == 0, no nothing + * @param memoryBuffer memoryBuffer which is writing + */ + private void end(MemoryBuffer memoryBuffer) { + if (chunkSize > 0) { + int currentWriteIndex = memoryBuffer.writerIndex(); + memoryBuffer.writerIndex(startOffset); + memoryBuffer.writeByte(chunkSize); + memoryBuffer.writeByte(header); + memoryBuffer.writerIndex(currentWriteIndex); + } + } + + /** + * if mark chunkFinish which means predict failed + * @param memoryBuffer memoryBuffer which is writing + */ + private void markChunkFinish(MemoryBuffer memoryBuffer) { + end(memoryBuffer); + memoryBuffer.writeByte(0); + } + + private void reset(MemoryBuffer memoryBuffer) { + end(memoryBuffer); + header = 0; + chunkSize = 0; + startOffset = memoryBuffer.writerIndex(); + memoryBuffer.writerIndex(startOffset + 2); + keyClass = null; + valueClass = null; + writeKeyClassInfo = false; + writeValueClassInfo = false; + } + + + } + + protected void generalJavaWriteV2(Fury fury, MemoryBuffer buffer, Map map) { + Chunk chunk = new Chunk(buffer, fury); + ClassResolver classResolver = fury.getClassResolver(); + RefResolver refResolver = fury.getRefResolver(); + boolean predict = true; + for (Object object : map.entrySet()) { + Map.Entry entry = (Map.Entry) object; + Object key = entry.getKey(); + Object value = entry.getValue(); + predict = predict && chunk.predict(key, value); + if (predict) { + chunk.write(key, value, buffer, keyClassInfoWriteCache, valueClassInfoWriteCache); + } else { + chunk.markChunkFinish(buffer); + writeJavaRefOptimized( + fury, classResolver, refResolver, buffer, entry.getKey(), keyClassInfoWriteCache); + writeJavaRefOptimized( + fury, classResolver, refResolver, buffer, entry.getValue(), valueClassInfoWriteCache); + } + } + chunk.end(buffer); + } + + public static void xwriteElements(Fury fury, MemoryBuffer buffer, Map value) { Generics generics = fury.getGenerics(); GenericType genericType = generics.nextGenericType(); @@ -440,14 +610,14 @@ private void genericJavaRead(Fury fury, MemoryBuffer buffer, Map map, int size) Generics generics = fury.getGenerics(); GenericType genericType = generics.nextGenericType(); if (genericType == null) { - generalJavaRead(fury, buffer, map, size); + generalJavaReadV2(fury, buffer, map, size); } else { GenericType keyGenericType = genericType.getTypeParameter0(); GenericType valueGenericType = genericType.getTypeParameter1(); if (genericType.getTypeParametersCount() < 2) { Tuple2 kvGenericType = getKVGenericType(genericType); if (keyGenericType == objType && valueGenericType == objType) { - generalJavaRead(fury, buffer, map, size); + generalJavaReadV2(fury, buffer, map, size); return; } keyGenericType = kvGenericType.f0; @@ -563,6 +733,57 @@ private void javaKVTypesNonFinalRead( } } + private void generalJavaReadV2(Fury fury, MemoryBuffer buffer, Map map, int size) { + int copySize = size; + while (copySize > 0) { + Object key = null; + Object value = null; + byte chunkSize = buffer.readByte(); + if (chunkSize == 0) { + key = fury.readRef(buffer, keyClassInfoReadCache); + value = fury.readRef(buffer, valueClassInfoReadCache); + } else { + byte header = buffer.readByte(); + Serializer keySerializer = fury.getClassResolver().readClassInfo(buffer, keyClassInfoReadCache).getSerializer(); + Serializer valueSerializer = fury.getClassResolver().readClassInfo(buffer, valueClassInfoReadCache).getSerializer(); + if ((header & MapFlags.TRACKING_KEY_REF) == MapFlags.TRACKING_KEY_REF) { + key = fury.readRef(buffer, keySerializer); + value = fury.readRef(buffer, valueSerializer); + } else { + if ((header & MapFlags.KEY_HAS_NULL) != MapFlags.KEY_HAS_NULL) { + key = keySerializer.read(buffer); + } else { + if ((header & MapFlags.VALUE_HAS_NULL) != MapFlags.VALUE_HAS_NULL) { + key = keySerializer.read(buffer); + } else { + if (buffer.readByte() == Fury.NULL_FLAG) { + key = null; + } else { + key = keySerializer.read(buffer); + } + } + } + } + if ((header & MapFlags.TRACKING_VALUE_REF) == MapFlags.TRACKING_VALUE_REF) { + + } else { + if ((header & MapFlags.VALUE_HAS_NULL) != MapFlags.VALUE_HAS_NULL) { + value = valueSerializer.read(buffer); + } else { + if (buffer.readByte() == Fury.NULL_FLAG) { + value = null; + } else { + value = valueSerializer.read(buffer); + } + } + } + } + //todo 1. object 2. fury.incDepth(1); + copySize--; + map.put(key, value); + } + } + private void generalJavaRead(Fury fury, MemoryBuffer buffer, Map map, int size) { for (int i = 0; i < size; i++) { Object key = fury.readRef(buffer, keyClassInfoReadCache); diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapFlags.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapFlags.java new file mode 100644 index 0000000000..3e76679b35 --- /dev/null +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapFlags.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fury.serializer.collection; + + +public class MapFlags { + + /** + * Whether track key ref. + */ + public static int TRACKING_KEY_REF = 0b1; + + /** + * Whether key has null. + */ + public static int KEY_HAS_NULL = 0b10; + + /** + * Whether key is not declare type. + */ + public static int KEY_NOT_DECL_TYPE = 0b100; + + /** + * Whether keys type are different. + */ + public static int KEY_NOT_SAME_TYPE = 0b1000; + + /** + * Whether track value ref. + */ + public static int TRACKING_VALUE_REF = 0b10000; + + /** + * Whether value has null. + */ + public static int VALUE_HAS_NULL = 0b100000; + + /** + * Whether value is not declare type. + */ + public static int VALUE_NOT_DECL_TYPE = 0b1000000; + + /** + * Whether values type are different. + */ + public static int VALUE_NOT_SAME_TYPE = 0b10000000; + + +} From 6e45eb68a9653763203b8dc4fd42cf06a7c9ca0a Mon Sep 17 00:00:00 2001 From: hening Date: Wed, 10 Jul 2024 10:00:01 +0800 Subject: [PATCH 02/18] map serializer chunk by chunk, key is not allowed null --- .../collection/AbstractMapSerializer.java | 161 +++++++++--------- 1 file changed, 76 insertions(+), 85 deletions(-) diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java index 7fc013d15f..b38560f138 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java @@ -161,7 +161,7 @@ private void genericJavaWrite(Fury fury, MemoryBuffer buffer, Map map) { Generics generics = fury.getGenerics(); GenericType genericType = generics.nextGenericType(); if (genericType == null) { - generalJavaWriteV2(fury, buffer, map); + javaChunkWrite(fury, buffer, map); } else { GenericType keyGenericType = genericType.getTypeParameter0(); GenericType valueGenericType = genericType.getTypeParameter1(); @@ -171,7 +171,7 @@ private void genericJavaWrite(Fury fury, MemoryBuffer buffer, Map map) { if (genericType.getTypeParametersCount() < 2) { Tuple2 kvGenericType = getKVGenericType(genericType); if (keyGenericType == objType && valueGenericType == objType) { - generalJavaWriteV2(fury, buffer, map); + javaChunkWrite(fury, buffer, map); return; } keyGenericType = kvGenericType.f0; @@ -330,18 +330,23 @@ private void generalJavaWrite(Fury fury, MemoryBuffer buffer, Map map) { } } - private static class Chunk { - - public Chunk(MemoryBuffer memoryBuffer, Fury fury) { + private static class ChunkWriter { + //todo hening remove the preserve two byte in constructor + public ChunkWriter(MemoryBuffer memoryBuffer, Fury fury) { final int writerIndex = memoryBuffer.writerIndex(); - // preserve a byte for header and chunk size + // preserve two byte for header and chunk size memoryBuffer.writerIndex(writerIndex + 2); this.startOffset = writerIndex; this.fury = fury; } public boolean predict(Object key, Object value) { - if (key == null || value == null) { + //key has null, stop predict + if (key == null) { + return false; + } + //ignore value has null, because null flag will be written + if (value == null) { return true; } if (keyClass == null) { @@ -362,12 +367,9 @@ public boolean predict(Object key, Object value) { private final Fury fury; private boolean writeKeyClassInfo = false; private boolean writeValueClassInfo = false; + private boolean markChunkFinish = false; public void write(Object key, Object value, MemoryBuffer memoryBuffer, ClassInfoHolder keyClassInfoWriteCache, ClassInfoHolder valueClassInfoWriteCache) { - //todo next chunk - if (chunkSize >= MAX_CHUNK_SIZE) { - reset(memoryBuffer); - } updateHeader(key, value, memoryBuffer, keyClassInfoWriteCache, valueClassInfoWriteCache); if ((header & MapFlags.TRACKING_KEY_REF) == MapFlags.TRACKING_KEY_REF) { RefResolver refResolver = fury.getRefResolver(); @@ -376,16 +378,7 @@ public void write(Object key, Object value, MemoryBuffer memoryBuffer, ClassInfo } } else { Serializer serializer = keyClassInfoWriteCache.getSerializer(); - if ((header & MapFlags.KEY_HAS_NULL) != MapFlags.KEY_HAS_NULL) { - serializer.write(memoryBuffer, key); - } else { - if (key == null) { - memoryBuffer.writeByte(Fury.NULL_FLAG); - } else { - memoryBuffer.writeByte(Fury.NOT_NULL_VALUE_FLAG); - serializer.write(memoryBuffer, key); - } - } + serializer.write(memoryBuffer, key); } if ((header & MapFlags.TRACKING_VALUE_REF) == MapFlags.TRACKING_VALUE_REF) { RefResolver refResolver = fury.getRefResolver(); @@ -394,22 +387,20 @@ public void write(Object key, Object value, MemoryBuffer memoryBuffer, ClassInfo } } else { Serializer serializer = valueClassInfoWriteCache.getSerializer(); - if ((header & MapFlags.VALUE_HAS_NULL) != MapFlags.VALUE_HAS_NULL) { - serializer.write(memoryBuffer, value); + if (value == null) { + memoryBuffer.writeByte(Fury.NULL_FLAG); } else { - if (value == null) { - memoryBuffer.writeByte(Fury.NULL_FLAG); - } else { - memoryBuffer.writeByte(Fury.NOT_NULL_VALUE_FLAG); - serializer.write(memoryBuffer, value); - } + memoryBuffer.writeByte(Fury.NOT_NULL_VALUE_FLAG); + serializer.write(memoryBuffer, value); } } chunkSize++; + if (chunkSize >= MAX_CHUNK_SIZE) { + reset(memoryBuffer); + } } public void updateHeader(Object key, Object value, MemoryBuffer memoryBuffer, ClassInfoHolder keyClassInfoWriteCache, ClassInfoHolder valueClassInfoWriteCache) { - //todo 如果key只有null if (key == null) { header |= MapFlags.KEY_HAS_NULL; } else { @@ -420,6 +411,7 @@ public void updateHeader(Object key, Object value, MemoryBuffer memoryBuffer, Cl } if (!writeKeyClassInfo) { classResolver.writeClass(memoryBuffer, classInfo); + writeKeyClassInfo = true; } } if (value == null) { @@ -432,21 +424,23 @@ public void updateHeader(Object key, Object value, MemoryBuffer memoryBuffer, Cl } if (!writeValueClassInfo) { classResolver.writeClass(memoryBuffer, classInfo); + writeValueClassInfo = true; } } } /** - * update chunk size and header, if chunk size == 0, no nothing + * update chunk size and header, if chunk size == 0, do nothing * @param memoryBuffer memoryBuffer which is writing */ - private void end(MemoryBuffer memoryBuffer) { + private void writeHeader(MemoryBuffer memoryBuffer) { if (chunkSize > 0) { int currentWriteIndex = memoryBuffer.writerIndex(); memoryBuffer.writerIndex(startOffset); memoryBuffer.writeByte(chunkSize); memoryBuffer.writeByte(header); memoryBuffer.writerIndex(currentWriteIndex); + chunkSize = 0; } } @@ -455,27 +449,28 @@ private void end(MemoryBuffer memoryBuffer) { * @param memoryBuffer memoryBuffer which is writing */ private void markChunkFinish(MemoryBuffer memoryBuffer) { - end(memoryBuffer); - memoryBuffer.writeByte(0); + if (!markChunkFinish) { + writeHeader(memoryBuffer); + memoryBuffer.writeByte(0); + markChunkFinish = true; + } } private void reset(MemoryBuffer memoryBuffer) { - end(memoryBuffer); + writeHeader(memoryBuffer); header = 0; chunkSize = 0; startOffset = memoryBuffer.writerIndex(); memoryBuffer.writerIndex(startOffset + 2); keyClass = null; valueClass = null; - writeKeyClassInfo = false; - writeValueClassInfo = false; } } - protected void generalJavaWriteV2(Fury fury, MemoryBuffer buffer, Map map) { - Chunk chunk = new Chunk(buffer, fury); + protected void javaChunkWrite(Fury fury, MemoryBuffer buffer, Map map) { + ChunkWriter chunkWriter = new ChunkWriter(buffer, fury); ClassResolver classResolver = fury.getClassResolver(); RefResolver refResolver = fury.getRefResolver(); boolean predict = true; @@ -483,18 +478,18 @@ protected void generalJavaWriteV2(Fury fury, MemoryBuffer buffer, Map map) { Map.Entry entry = (Map.Entry) object; Object key = entry.getKey(); Object value = entry.getValue(); - predict = predict && chunk.predict(key, value); + predict = predict && chunkWriter.predict(key, value); if (predict) { - chunk.write(key, value, buffer, keyClassInfoWriteCache, valueClassInfoWriteCache); + chunkWriter.write(key, value, buffer, keyClassInfoWriteCache, valueClassInfoWriteCache); } else { - chunk.markChunkFinish(buffer); + chunkWriter.markChunkFinish(buffer); writeJavaRefOptimized( fury, classResolver, refResolver, buffer, entry.getKey(), keyClassInfoWriteCache); writeJavaRefOptimized( fury, classResolver, refResolver, buffer, entry.getValue(), valueClassInfoWriteCache); } } - chunk.end(buffer); + chunkWriter.writeHeader(buffer); } @@ -610,14 +605,14 @@ private void genericJavaRead(Fury fury, MemoryBuffer buffer, Map map, int size) Generics generics = fury.getGenerics(); GenericType genericType = generics.nextGenericType(); if (genericType == null) { - generalJavaReadV2(fury, buffer, map, size); + javaChunkRead(fury, buffer, map, size); } else { GenericType keyGenericType = genericType.getTypeParameter0(); GenericType valueGenericType = genericType.getTypeParameter1(); if (genericType.getTypeParametersCount() < 2) { Tuple2 kvGenericType = getKVGenericType(genericType); if (keyGenericType == objType && valueGenericType == objType) { - generalJavaReadV2(fury, buffer, map, size); + javaChunkRead(fury, buffer, map, size); return; } keyGenericType = kvGenericType.f0; @@ -733,55 +728,51 @@ private void javaKVTypesNonFinalRead( } } - private void generalJavaReadV2(Fury fury, MemoryBuffer buffer, Map map, int size) { + private void javaChunkRead(Fury fury, MemoryBuffer buffer, Map map, int size) { int copySize = size; - while (copySize > 0) { - Object key = null; + Serializer keySerializer = null; + Serializer valueSerializer = null; + byte chunkSize = buffer.readByte(); + if (chunkSize == 0) { + generalJavaRead(fury, buffer, map, size); + return; + } + byte header = buffer.readByte(); + while (copySize > 0 && chunkSize > 0) { + Object key; Object value = null; - byte chunkSize = buffer.readByte(); - if (chunkSize == 0) { - key = fury.readRef(buffer, keyClassInfoReadCache); - value = fury.readRef(buffer, valueClassInfoReadCache); + if (keySerializer == null) { + keySerializer = fury.getClassResolver().readClassInfo(buffer, keyClassInfoReadCache).getSerializer(); + } + if (valueSerializer == null) { + valueSerializer = fury.getClassResolver().readClassInfo(buffer, valueClassInfoReadCache).getSerializer(); + } + if ((header & MapFlags.TRACKING_KEY_REF) == MapFlags.TRACKING_KEY_REF) { + key = fury.readRef(buffer, keySerializer); } else { - byte header = buffer.readByte(); - Serializer keySerializer = fury.getClassResolver().readClassInfo(buffer, keyClassInfoReadCache).getSerializer(); - Serializer valueSerializer = fury.getClassResolver().readClassInfo(buffer, valueClassInfoReadCache).getSerializer(); - if ((header & MapFlags.TRACKING_KEY_REF) == MapFlags.TRACKING_KEY_REF) { - key = fury.readRef(buffer, keySerializer); - value = fury.readRef(buffer, valueSerializer); - } else { - if ((header & MapFlags.KEY_HAS_NULL) != MapFlags.KEY_HAS_NULL) { - key = keySerializer.read(buffer); - } else { - if ((header & MapFlags.VALUE_HAS_NULL) != MapFlags.VALUE_HAS_NULL) { - key = keySerializer.read(buffer); - } else { - if (buffer.readByte() == Fury.NULL_FLAG) { - key = null; - } else { - key = keySerializer.read(buffer); - } - } - } - } - if ((header & MapFlags.TRACKING_VALUE_REF) == MapFlags.TRACKING_VALUE_REF) { - - } else { - if ((header & MapFlags.VALUE_HAS_NULL) != MapFlags.VALUE_HAS_NULL) { - value = valueSerializer.read(buffer); - } else { - if (buffer.readByte() == Fury.NULL_FLAG) { - value = null; - } else { - value = valueSerializer.read(buffer); - } - } + key = keySerializer.read(buffer); + } + if ((header & MapFlags.TRACKING_VALUE_REF) == MapFlags.TRACKING_VALUE_REF) { + value = fury.readRef(buffer, valueSerializer); + } else { + if (buffer.readByte() != Fury.NULL_FLAG) { + value = valueSerializer.read(buffer); } } //todo 1. object 2. fury.incDepth(1); copySize--; + chunkSize--; map.put(key, value); + if (chunkSize == 0 && copySize != 0) { + chunkSize = buffer.readByte(); + if (chunkSize == 0) { + generalJavaRead(fury, buffer, map, copySize); + } else { + header = buffer.readByte(); + } + } } + } private void generalJavaRead(Fury fury, MemoryBuffer buffer, Map map, int size) { From 7fd2dc62ff090a0ebebd6ae99ba7a8e707bd0f99 Mon Sep 17 00:00:00 2001 From: hening Date: Wed, 10 Jul 2024 14:20:45 +0800 Subject: [PATCH 03/18] preserveByteForHeaderAndChunkSize and extract method --- .../collection/AbstractMapSerializer.java | 97 ++++++++++++------- 1 file changed, 61 insertions(+), 36 deletions(-) diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java index b38560f138..3c16147c26 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java @@ -331,12 +331,8 @@ private void generalJavaWrite(Fury fury, MemoryBuffer buffer, Map map) { } private static class ChunkWriter { - //todo hening remove the preserve two byte in constructor - public ChunkWriter(MemoryBuffer memoryBuffer, Fury fury) { - final int writerIndex = memoryBuffer.writerIndex(); - // preserve two byte for header and chunk size - memoryBuffer.writerIndex(writerIndex + 2); - this.startOffset = writerIndex; + + public ChunkWriter(Fury fury) { this.fury = fury; } @@ -367,10 +363,38 @@ public boolean predict(Object key, Object value) { private final Fury fury; private boolean writeKeyClassInfo = false; private boolean writeValueClassInfo = false; - private boolean markChunkFinish = false; + /** + * mark chunk write finish, + */ + private boolean markChunkWriteFinish = false; + /** + * preserve two byte for header and chunk size and record the write index + * so that we can write key value at first, write header and chunk size when the chunk is finish at correct position + */ + private boolean preserveByteForHeaderAndChunkSize = false; + + private void preserveByteForHeaderAndChunkSize(MemoryBuffer memoryBuffer) { + if (!preserveByteForHeaderAndChunkSize) { + int writerIndex = memoryBuffer.writerIndex(); + // preserve two byte for header and chunk size + memoryBuffer.writerIndex(writerIndex + 2); + this.startOffset = writerIndex; + preserveByteForHeaderAndChunkSize = true; + } + } public void write(Object key, Object value, MemoryBuffer memoryBuffer, ClassInfoHolder keyClassInfoWriteCache, ClassInfoHolder valueClassInfoWriteCache) { + preserveByteForHeaderAndChunkSize(memoryBuffer); updateHeader(key, value, memoryBuffer, keyClassInfoWriteCache, valueClassInfoWriteCache); + writeKey(key, memoryBuffer, keyClassInfoWriteCache); + writeValue(value, memoryBuffer, valueClassInfoWriteCache); + chunkSize++; + if (chunkSize >= MAX_CHUNK_SIZE) { + reset(memoryBuffer); + } + } + + private void writeKey(Object key, MemoryBuffer memoryBuffer, ClassInfoHolder keyClassInfoWriteCache) { if ((header & MapFlags.TRACKING_KEY_REF) == MapFlags.TRACKING_KEY_REF) { RefResolver refResolver = fury.getRefResolver(); if (!refResolver.writeRefOrNull(memoryBuffer, key)) { @@ -380,25 +404,24 @@ public void write(Object key, Object value, MemoryBuffer memoryBuffer, ClassInfo Serializer serializer = keyClassInfoWriteCache.getSerializer(); serializer.write(memoryBuffer, key); } - if ((header & MapFlags.TRACKING_VALUE_REF) == MapFlags.TRACKING_VALUE_REF) { - RefResolver refResolver = fury.getRefResolver(); - if (!refResolver.writeRefOrNull(memoryBuffer, key)) { - valueClassInfoWriteCache.getSerializer().write(memoryBuffer, key); - } - } else { - Serializer serializer = valueClassInfoWriteCache.getSerializer(); - if (value == null) { - memoryBuffer.writeByte(Fury.NULL_FLAG); + } + + private void writeValue(Object value, MemoryBuffer memoryBuffer, ClassInfoHolder valueClassInfoWriteCache) { + if ((header & MapFlags.TRACKING_VALUE_REF) == MapFlags.TRACKING_VALUE_REF) { + RefResolver refResolver = fury.getRefResolver(); + if (!refResolver.writeRefOrNull(memoryBuffer, value)) { + valueClassInfoWriteCache.getSerializer().write(memoryBuffer, value); + } } else { - memoryBuffer.writeByte(Fury.NOT_NULL_VALUE_FLAG); - serializer.write(memoryBuffer, value); + Serializer serializer = valueClassInfoWriteCache.getSerializer(); + if (value == null) { + memoryBuffer.writeByte(Fury.NULL_FLAG); + } else { + memoryBuffer.writeByte(Fury.NOT_NULL_VALUE_FLAG); + serializer.write(memoryBuffer, value); + } } } - chunkSize++; - if (chunkSize >= MAX_CHUNK_SIZE) { - reset(memoryBuffer); - } - } public void updateHeader(Object key, Object value, MemoryBuffer memoryBuffer, ClassInfoHolder keyClassInfoWriteCache, ClassInfoHolder valueClassInfoWriteCache) { if (key == null) { @@ -431,7 +454,7 @@ public void updateHeader(Object key, Object value, MemoryBuffer memoryBuffer, Cl /** * update chunk size and header, if chunk size == 0, do nothing - * @param memoryBuffer memoryBuffer which is writing + * @param memoryBuffer memoryBuffer which is written */ private void writeHeader(MemoryBuffer memoryBuffer) { if (chunkSize > 0) { @@ -445,32 +468,34 @@ private void writeHeader(MemoryBuffer memoryBuffer) { } /** - * if mark chunkFinish which means predict failed - * @param memoryBuffer memoryBuffer which is writing + * use chunk size = 0 to mark chunk write finish, + * if mark chunk write finish which means predict failed, chunk write is finish, + * rest of map will be written by generalJavaWrite + * @param memoryBuffer memoryBuffer which is written */ - private void markChunkFinish(MemoryBuffer memoryBuffer) { - if (!markChunkFinish) { + private void markChunkWriteFinish(MemoryBuffer memoryBuffer) { + if (!markChunkWriteFinish) { writeHeader(memoryBuffer); memoryBuffer.writeByte(0); - markChunkFinish = true; + markChunkWriteFinish = true; } } + /** + * chunk size reach max size, start new chunk, no need reset keyClass and value Class + * @param memoryBuffer memoryBuffer which is written + */ private void reset(MemoryBuffer memoryBuffer) { writeHeader(memoryBuffer); header = 0; chunkSize = 0; - startOffset = memoryBuffer.writerIndex(); - memoryBuffer.writerIndex(startOffset + 2); - keyClass = null; - valueClass = null; + preserveByteForHeaderAndChunkSize = false; } - } protected void javaChunkWrite(Fury fury, MemoryBuffer buffer, Map map) { - ChunkWriter chunkWriter = new ChunkWriter(buffer, fury); + ChunkWriter chunkWriter = new ChunkWriter(fury); ClassResolver classResolver = fury.getClassResolver(); RefResolver refResolver = fury.getRefResolver(); boolean predict = true; @@ -482,7 +507,7 @@ protected void javaChunkWrite(Fury fury, MemoryBuffer buffer, Map map) { if (predict) { chunkWriter.write(key, value, buffer, keyClassInfoWriteCache, valueClassInfoWriteCache); } else { - chunkWriter.markChunkFinish(buffer); + chunkWriter.markChunkWriteFinish(buffer); writeJavaRefOptimized( fury, classResolver, refResolver, buffer, entry.getKey(), keyClassInfoWriteCache); writeJavaRefOptimized( From 9242f7680271a3a7b075c2e814fb879d44325774 Mon Sep 17 00:00:00 2001 From: hening Date: Wed, 10 Jul 2024 14:57:52 +0800 Subject: [PATCH 04/18] checkChunkSize and modify max chunk size to 127 --- .../serializer/collection/AbstractMapSerializer.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java index 3c16147c26..a05cc1ecaf 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java @@ -41,7 +41,7 @@ /** Serializer for all map-like objects. */ @SuppressWarnings({"unchecked", "rawtypes"}) public abstract class AbstractMapSerializer extends Serializer { - private static final int MAX_CHUNK_SIZE = 128; + private static final int MAX_CHUNK_SIZE = 127; protected MethodHandle constructor; protected final boolean supportCodegenHook; private Serializer keySerializer; @@ -753,11 +753,18 @@ private void javaKVTypesNonFinalRead( } } + private static void checkChunkSize(byte chunkSize) { + if (chunkSize < 0) { + throw new RuntimeException("chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); + } + } + private void javaChunkRead(Fury fury, MemoryBuffer buffer, Map map, int size) { int copySize = size; Serializer keySerializer = null; Serializer valueSerializer = null; byte chunkSize = buffer.readByte(); + checkChunkSize(chunkSize); if (chunkSize == 0) { generalJavaRead(fury, buffer, map, size); return; @@ -790,6 +797,7 @@ private void javaChunkRead(Fury fury, MemoryBuffer buffer, Map map, int size) { map.put(key, value); if (chunkSize == 0 && copySize != 0) { chunkSize = buffer.readByte(); + checkChunkSize(chunkSize); if (chunkSize == 0) { generalJavaRead(fury, buffer, map, copySize); } else { From 7fbe54dd80e4f3de07bd95b001e6dd540b36a494 Mon Sep 17 00:00:00 2001 From: hening Date: Thu, 11 Jul 2024 14:37:56 +0800 Subject: [PATCH 05/18] use Preconditions check chunk size --- .../serializer/collection/AbstractMapSerializer.java | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java index a05cc1ecaf..fee5996bca 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java @@ -37,6 +37,7 @@ import org.apache.fury.type.GenericType; import org.apache.fury.type.Generics; import org.apache.fury.type.TypeUtils; +import org.apache.fury.util.Preconditions; /** Serializer for all map-like objects. */ @SuppressWarnings({"unchecked", "rawtypes"}) @@ -753,18 +754,13 @@ private void javaKVTypesNonFinalRead( } } - private static void checkChunkSize(byte chunkSize) { - if (chunkSize < 0) { - throw new RuntimeException("chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); - } - } private void javaChunkRead(Fury fury, MemoryBuffer buffer, Map map, int size) { int copySize = size; Serializer keySerializer = null; Serializer valueSerializer = null; byte chunkSize = buffer.readByte(); - checkChunkSize(chunkSize); + Preconditions.checkArgument(chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); if (chunkSize == 0) { generalJavaRead(fury, buffer, map, size); return; @@ -797,7 +793,7 @@ private void javaChunkRead(Fury fury, MemoryBuffer buffer, Map map, int size) { map.put(key, value); if (chunkSize == 0 && copySize != 0) { chunkSize = buffer.readByte(); - checkChunkSize(chunkSize); + Preconditions.checkArgument(chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); if (chunkSize == 0) { generalJavaRead(fury, buffer, map, copySize); } else { From 07d79118f159497a0eb035a1a7833d49e47b6a9a Mon Sep 17 00:00:00 2001 From: hening Date: Fri, 26 Jul 2024 15:05:31 +0800 Subject: [PATCH 06/18] save code --- .../collection/AbstractMapSerializer.java | 448 +++++++-------- .../serializer/collection/MapChunkWriter.java | 529 ++++++++++++++++++ .../fury/serializer/collection/MapFlags.java | 18 +- .../collection/MapSerializersTest.java | 3 +- 4 files changed, 733 insertions(+), 265 deletions(-) create mode 100644 java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapChunkWriter.java diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java index fee5996bca..8f6bc05ad5 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java @@ -29,7 +29,6 @@ import org.apache.fury.memory.MemoryBuffer; import org.apache.fury.reflect.ReflectionUtils; import org.apache.fury.reflect.TypeRef; -import org.apache.fury.resolver.ClassInfo; import org.apache.fury.resolver.ClassInfoHolder; import org.apache.fury.resolver.ClassResolver; import org.apache.fury.resolver.RefResolver; @@ -42,7 +41,6 @@ /** Serializer for all map-like objects. */ @SuppressWarnings({"unchecked", "rawtypes"}) public abstract class AbstractMapSerializer extends Serializer { - private static final int MAX_CHUNK_SIZE = 127; protected MethodHandle constructor; protected final boolean supportCodegenHook; private Serializer keySerializer; @@ -117,30 +115,44 @@ protected final void writeElements(Fury fury, MemoryBuffer buffer, Map map) { this.keySerializer = null; this.valueSerializer = null; if (keySerializer != null && valueSerializer != null) { - javaWriteWithKVSerializers(fury, buffer, map, keySerializer, valueSerializer); + javaWriteWithKVSerializers(fury, buffer, map, keySerializer, valueSerializer); } else if (keySerializer != null) { + javaWriteWithKeySerializers(map, buffer, keySerializer); + } else if (valueSerializer != null) { + javaWriteWithValueSerializers(map, buffer, valueSerializer); + } else { + genericJavaWrite(fury, buffer, map); + } + } + + private void javaWriteWithKeySerializers(Map map, MemoryBuffer buffer, Serializer keySerializer) { + MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); ClassResolver classResolver = fury.getClassResolver(); RefResolver refResolver = fury.getRefResolver(); for (Object object : map.entrySet()) { - Map.Entry entry = (Map.Entry) object; - fury.writeRef(buffer, entry.getKey(), keySerializer); - Object value = entry.getValue(); - writeJavaRefOptimized( - fury, classResolver, refResolver, buffer, value, valueClassInfoWriteCache); + Map.Entry entry = (Map.Entry) object; + mapChunkWriter.writeFinalKey(entry.getKey(), buffer, keySerializer); + Object value = entry.getValue(); + mapChunkWriter.writeValue(value, buffer, classResolver, refResolver, valueClassInfoWriteCache); + mapChunkWriter.increaseChunkSize(); + mapChunkWriter.resetIfNeed(entry.getKey(), buffer); } - } else if (valueSerializer != null) { + mapChunkWriter.writeHeader(buffer); + } + + private void javaWriteWithValueSerializers(Map map, MemoryBuffer buffer, Serializer valueSerializer) { + MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); ClassResolver classResolver = fury.getClassResolver(); RefResolver refResolver = fury.getRefResolver(); for (Object object : map.entrySet()) { - Map.Entry entry = (Map.Entry) object; - Object key = entry.getKey(); - writeJavaRefOptimized( - fury, classResolver, refResolver, buffer, key, keyClassInfoWriteCache); - fury.writeRef(buffer, entry.getValue(), valueSerializer); + Map.Entry entry = (Map.Entry) object; + Object key = entry.getKey(); + mapChunkWriter.writeKey(key, buffer, classResolver, refResolver, keyClassInfoWriteCache); + mapChunkWriter.writeFinalValue(entry.getValue(), buffer, valueSerializer); + mapChunkWriter.increaseChunkSize(); + mapChunkWriter.resetIfNeed(key, buffer); } - } else { - genericJavaWrite(fury, buffer, map); - } + mapChunkWriter.writeHeader(buffer); } private void javaWriteWithKVSerializers( @@ -149,13 +161,19 @@ private void javaWriteWithKVSerializers( Map map, Serializer keySerializer, Serializer valueSerializer) { + MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); for (Object object : map.entrySet()) { Map.Entry entry = (Map.Entry) object; Object key = entry.getKey(); Object value = entry.getValue(); - fury.writeRef(buffer, key, keySerializer); - fury.writeRef(buffer, value, valueSerializer); + mapChunkWriter.writeFinalValue(value, buffer, valueSerializer); + mapChunkWriter.writeFinalKey(key, buffer, keySerializer); + mapChunkWriter.increaseChunkSize(); + mapChunkWriter.resetIfNeed(key, buffer); +// fury.writeRef(buffer, key, keySerializer); +// fury.writeRef(buffer, value, valueSerializer); } + mapChunkWriter.writeHeader(buffer); } private void genericJavaWrite(Fury fury, MemoryBuffer buffer, Map map) { @@ -194,13 +212,13 @@ private void genericJavaWrite(Fury fury, MemoryBuffer buffer, Map map) { boolean keyGenericTypeFinal = keyGenericType.isMonomorphic(); boolean valueGenericTypeFinal = valueGenericType.isMonomorphic(); if (keyGenericTypeFinal && valueGenericTypeFinal) { - javaKVTypesFinalWrite(fury, buffer, map, keyGenericType, valueGenericType, generics); + javaKVTypesFinalChunkWrite(fury, buffer, map, keyGenericType, valueGenericType, generics); } else if (keyGenericTypeFinal) { - javaKeyTypeFinalWrite(fury, buffer, map, keyGenericType, valueGenericType, generics); + javaKeyTypeFinalChunkWrite(fury, buffer, map, keyGenericType, valueGenericType, generics); } else if (valueGenericTypeFinal) { - javaValueTypeFinalWrite(fury, buffer, map, keyGenericType, valueGenericType, generics); + javaValueTypeFinalChunkWrite(fury, buffer, map, keyGenericType, valueGenericType, generics); } else { - javaKVTypesNonFinalWrite(fury, buffer, map, keyGenericType, valueGenericType, generics); + javaKVTypesNonFinalChunkWrite(fury, buffer, map, keyGenericType, valueGenericType, generics); } } } @@ -225,6 +243,42 @@ private void javaKVTypesFinalWrite( } } + /** + * kv final write do not need to predict , since key and value is almost same type unless null + * + * @param fury + * @param buffer + * @param map + * @param keyGenericType + * @param valueGenericType + * @param generics + */ + private void javaKVTypesFinalChunkWrite( + Fury fury, + MemoryBuffer buffer, + Map map, + GenericType keyGenericType, + GenericType valueGenericType, + Generics generics) { + MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); + Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); + Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); + for (Object object : map.entrySet()) { + Map.Entry entry = (Map.Entry) object; + Object key = entry.getKey(); + Object value = entry.getValue(); + generics.pushGenericType(keyGenericType); + mapChunkWriter.writeFinalKey(key, buffer, keySerializer); + generics.popGenericType(); + generics.pushGenericType(valueGenericType); + mapChunkWriter.writeFinalValue(value, buffer, valueSerializer); + generics.popGenericType(); + mapChunkWriter.increaseChunkSize(); + mapChunkWriter.resetIfNeed(key, buffer); + } + mapChunkWriter.writeHeader(buffer); + } + private void javaKeyTypeFinalWrite( Fury fury, MemoryBuffer buffer, @@ -254,69 +308,85 @@ private void javaKeyTypeFinalWrite( } } - private void javaValueTypeFinalWrite( + private void javaKeyTypeFinalChunkWrite( Fury fury, MemoryBuffer buffer, Map map, GenericType keyGenericType, GenericType valueGenericType, Generics generics) { - ClassResolver classResolver = fury.getClassResolver(); - RefResolver refResolver = fury.getRefResolver(); - boolean trackingKeyRef = fury.getClassResolver().needToWriteRef(keyGenericType.getCls()); - Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); - for (Object object : map.entrySet()) { - Map.Entry entry = (Map.Entry) object; - generics.pushGenericType(keyGenericType); - writeJavaRefOptimized( - fury, - classResolver, - refResolver, - trackingKeyRef, - buffer, - entry.getKey(), - keyClassInfoWriteCache); - generics.popGenericType(); - generics.pushGenericType(valueGenericType); - fury.writeRef(buffer, entry.getValue(), valueSerializer); - generics.popGenericType(); - } + MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); + ClassResolver classResolver = fury.getClassResolver(); + RefResolver refResolver = fury.getRefResolver(); + Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); + for (Object object : map.entrySet()) { + Map.Entry entry = (Map.Entry) object; + Object key = entry.getKey(); + Object value = entry.getValue(); + generics.pushGenericType(keyGenericType); + mapChunkWriter.writeFinalKey(key, buffer, keySerializer); + generics.popGenericType(); + generics.pushGenericType(valueGenericType); + mapChunkWriter.writeValueWithGenericType(value, buffer, classResolver, refResolver, valueGenericType, valueClassInfoWriteCache); + generics.popGenericType(); + mapChunkWriter.increaseChunkSize(); + mapChunkWriter.resetIfNeed(key, buffer); + } + mapChunkWriter.writeHeader(buffer); } - private void javaKVTypesNonFinalWrite( + private void javaValueTypeFinalChunkWrite( Fury fury, MemoryBuffer buffer, Map map, GenericType keyGenericType, GenericType valueGenericType, Generics generics) { - ClassResolver classResolver = fury.getClassResolver(); + final MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); + ClassResolver classResolver = fury.getClassResolver(); + RefResolver refResolver = fury.getRefResolver(); + Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); + for (Object object : map.entrySet()) { + Map.Entry entry = (Map.Entry) object; + Object key = entry.getKey(); + Object value = entry.getValue(); + generics.pushGenericType(keyGenericType); + mapChunkWriter.writeKeyWithGenericType(key, buffer, classResolver, refResolver, keyGenericType, keyClassInfoWriteCache); + generics.popGenericType(); + generics.pushGenericType(valueGenericType); + mapChunkWriter.writeFinalValue(value, buffer, valueSerializer); + generics.popGenericType(); + mapChunkWriter.increaseChunkSize(); + mapChunkWriter.resetIfNeed(key, buffer); + } + mapChunkWriter.writeHeader(buffer); + } + + private void javaKVTypesNonFinalChunkWrite( + Fury fury, + MemoryBuffer buffer, + Map map, + GenericType keyGenericType, + GenericType valueGenericType, + Generics generics) { + MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); + ClassResolver classResolver = fury.getClassResolver(); RefResolver refResolver = fury.getRefResolver(); - boolean trackingKeyRef = fury.getClassResolver().needToWriteRef(keyGenericType.getCls()); boolean trackingValueRef = fury.getClassResolver().needToWriteRef(valueGenericType.getCls()); for (Object object : map.entrySet()) { - Map.Entry entry = (Map.Entry) object; - generics.pushGenericType(keyGenericType); - writeJavaRefOptimized( - fury, - classResolver, - refResolver, - trackingKeyRef, - buffer, - entry.getKey(), - keyClassInfoWriteCache); - generics.popGenericType(); - generics.pushGenericType(valueGenericType); - writeJavaRefOptimized( - fury, - classResolver, - refResolver, - trackingValueRef, - buffer, - entry.getValue(), - valueClassInfoWriteCache); - generics.popGenericType(); + Map.Entry entry = (Map.Entry) object; + Object key = entry.getKey(); + Object value = entry.getValue(); + generics.pushGenericType(keyGenericType); + mapChunkWriter.writeKeyWithGenericType(key, buffer, classResolver, refResolver, keyGenericType, keyClassInfoWriteCache); + generics.popGenericType(); + generics.pushGenericType(valueGenericType); + mapChunkWriter.writeValue(value, buffer, classResolver, refResolver, valueClassInfoWriteCache); + generics.popGenericType(); + mapChunkWriter.increaseChunkSize(); + mapChunkWriter.resetIfNeed(entry.getKey(), buffer); } + mapChunkWriter.writeHeader(buffer); } private void generalJavaWrite(Fury fury, MemoryBuffer buffer, Map map) { @@ -331,191 +401,16 @@ private void generalJavaWrite(Fury fury, MemoryBuffer buffer, Map map) { } } - private static class ChunkWriter { - - public ChunkWriter(Fury fury) { - this.fury = fury; - } - - public boolean predict(Object key, Object value) { - //key has null, stop predict - if (key == null) { - return false; - } - //ignore value has null, because null flag will be written - if (value == null) { - return true; - } - if (keyClass == null) { - keyClass = key.getClass(); - } - if (valueClass == null) { - valueClass = value.getClass(); - } - return key.getClass() == keyClass - && value.getClass() == valueClass; - } - - private int header = 0; - private int startOffset; - private int chunkSize; - private Class keyClass = null; - private Class valueClass = null; - private final Fury fury; - private boolean writeKeyClassInfo = false; - private boolean writeValueClassInfo = false; - /** - * mark chunk write finish, - */ - private boolean markChunkWriteFinish = false; - /** - * preserve two byte for header and chunk size and record the write index - * so that we can write key value at first, write header and chunk size when the chunk is finish at correct position - */ - private boolean preserveByteForHeaderAndChunkSize = false; - - private void preserveByteForHeaderAndChunkSize(MemoryBuffer memoryBuffer) { - if (!preserveByteForHeaderAndChunkSize) { - int writerIndex = memoryBuffer.writerIndex(); - // preserve two byte for header and chunk size - memoryBuffer.writerIndex(writerIndex + 2); - this.startOffset = writerIndex; - preserveByteForHeaderAndChunkSize = true; - } - } - - public void write(Object key, Object value, MemoryBuffer memoryBuffer, ClassInfoHolder keyClassInfoWriteCache, ClassInfoHolder valueClassInfoWriteCache) { - preserveByteForHeaderAndChunkSize(memoryBuffer); - updateHeader(key, value, memoryBuffer, keyClassInfoWriteCache, valueClassInfoWriteCache); - writeKey(key, memoryBuffer, keyClassInfoWriteCache); - writeValue(value, memoryBuffer, valueClassInfoWriteCache); - chunkSize++; - if (chunkSize >= MAX_CHUNK_SIZE) { - reset(memoryBuffer); - } - } - - private void writeKey(Object key, MemoryBuffer memoryBuffer, ClassInfoHolder keyClassInfoWriteCache) { - if ((header & MapFlags.TRACKING_KEY_REF) == MapFlags.TRACKING_KEY_REF) { - RefResolver refResolver = fury.getRefResolver(); - if (!refResolver.writeRefOrNull(memoryBuffer, key)) { - keyClassInfoWriteCache.getSerializer().write(memoryBuffer, key); - } - } else { - Serializer serializer = keyClassInfoWriteCache.getSerializer(); - serializer.write(memoryBuffer, key); - } - } - - private void writeValue(Object value, MemoryBuffer memoryBuffer, ClassInfoHolder valueClassInfoWriteCache) { - if ((header & MapFlags.TRACKING_VALUE_REF) == MapFlags.TRACKING_VALUE_REF) { - RefResolver refResolver = fury.getRefResolver(); - if (!refResolver.writeRefOrNull(memoryBuffer, value)) { - valueClassInfoWriteCache.getSerializer().write(memoryBuffer, value); - } - } else { - Serializer serializer = valueClassInfoWriteCache.getSerializer(); - if (value == null) { - memoryBuffer.writeByte(Fury.NULL_FLAG); - } else { - memoryBuffer.writeByte(Fury.NOT_NULL_VALUE_FLAG); - serializer.write(memoryBuffer, value); - } - } - } - - public void updateHeader(Object key, Object value, MemoryBuffer memoryBuffer, ClassInfoHolder keyClassInfoWriteCache, ClassInfoHolder valueClassInfoWriteCache) { - if (key == null) { - header |= MapFlags.KEY_HAS_NULL; - } else { - ClassResolver classResolver = fury.getClassResolver(); - ClassInfo classInfo = classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache); - if (classInfo.getSerializer().needToWriteRef()) { - header |= MapFlags.TRACKING_KEY_REF; - } - if (!writeKeyClassInfo) { - classResolver.writeClass(memoryBuffer, classInfo); - writeKeyClassInfo = true; - } - } - if (value == null) { - header |= MapFlags.VALUE_HAS_NULL; - } else { - ClassResolver classResolver = fury.getClassResolver(); - ClassInfo classInfo = classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache); - if (classInfo.getSerializer().needToWriteRef()) { - header |= MapFlags.TRACKING_VALUE_REF; - } - if (!writeValueClassInfo) { - classResolver.writeClass(memoryBuffer, classInfo); - writeValueClassInfo = true; - } - } - } - - /** - * update chunk size and header, if chunk size == 0, do nothing - * @param memoryBuffer memoryBuffer which is written - */ - private void writeHeader(MemoryBuffer memoryBuffer) { - if (chunkSize > 0) { - int currentWriteIndex = memoryBuffer.writerIndex(); - memoryBuffer.writerIndex(startOffset); - memoryBuffer.writeByte(chunkSize); - memoryBuffer.writeByte(header); - memoryBuffer.writerIndex(currentWriteIndex); - chunkSize = 0; - } - } - - /** - * use chunk size = 0 to mark chunk write finish, - * if mark chunk write finish which means predict failed, chunk write is finish, - * rest of map will be written by generalJavaWrite - * @param memoryBuffer memoryBuffer which is written - */ - private void markChunkWriteFinish(MemoryBuffer memoryBuffer) { - if (!markChunkWriteFinish) { - writeHeader(memoryBuffer); - memoryBuffer.writeByte(0); - markChunkWriteFinish = true; - } - } - - /** - * chunk size reach max size, start new chunk, no need reset keyClass and value Class - * @param memoryBuffer memoryBuffer which is written - */ - private void reset(MemoryBuffer memoryBuffer) { - writeHeader(memoryBuffer); - header = 0; - chunkSize = 0; - preserveByteForHeaderAndChunkSize = false; - } - - } - protected void javaChunkWrite(Fury fury, MemoryBuffer buffer, Map map) { - ChunkWriter chunkWriter = new ChunkWriter(fury); + MapChunkWriter chunkWriter = new MapChunkWriter(fury); ClassResolver classResolver = fury.getClassResolver(); RefResolver refResolver = fury.getRefResolver(); - boolean predict = true; for (Object object : map.entrySet()) { - Map.Entry entry = (Map.Entry) object; - Object key = entry.getKey(); - Object value = entry.getValue(); - predict = predict && chunkWriter.predict(key, value); - if (predict) { - chunkWriter.write(key, value, buffer, keyClassInfoWriteCache, valueClassInfoWriteCache); - } else { - chunkWriter.markChunkWriteFinish(buffer); - writeJavaRefOptimized( - fury, classResolver, refResolver, buffer, entry.getKey(), keyClassInfoWriteCache); - writeJavaRefOptimized( - fury, classResolver, refResolver, buffer, entry.getValue(), valueClassInfoWriteCache); - } + Map.Entry entry = (Map.Entry) object; + Object key = entry.getKey(); + Object value = entry.getValue(); + chunkWriter.generalChunkWrite(key, value, buffer, classResolver, refResolver, keyClassInfoWriteCache, valueClassInfoWriteCache); } - chunkWriter.writeHeader(buffer); } @@ -606,11 +501,7 @@ protected final void readElements(MemoryBuffer buffer, int size, Map map) { this.keySerializer = null; this.valueSerializer = null; if (keySerializer != null && valueSerializer != null) { - for (int i = 0; i < size; i++) { - Object key = fury.readRef(buffer, keySerializer); - Object value = fury.readRef(buffer, valueSerializer); - map.put(key, value); - } + javaChunkReadWithKVSerializers(buffer, map, size, keySerializer, valueSerializer); } else if (keySerializer != null) { for (int i = 0; i < size; i++) { Object key = fury.readRef(buffer, keySerializer); @@ -627,6 +518,23 @@ protected final void readElements(MemoryBuffer buffer, int size, Map map) { } } + private void javaChunkReadWithKVSerializers(MemoryBuffer buffer, Map map, int size, Serializer keySerializer, Serializer valueSerializer) { + MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); + while (size > 0) { + byte chunkSize = buffer.readByte(); + byte header = buffer.readByte(); + Preconditions.checkArgument(chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); + for (byte i = 0; i < chunkSize; i++) { + Object key; + Object value; + key = mapChunkWriter.readFinalKey(buffer, header, keySerializer); + value = mapChunkWriter.readFinalValue(buffer, header, valueSerializer); + map.put(key, value); + size--; + } + } + } + private void genericJavaRead(Fury fury, MemoryBuffer buffer, Map map, int size) { Generics generics = fury.getGenerics(); GenericType genericType = generics.nextGenericType(); @@ -638,7 +546,7 @@ private void genericJavaRead(Fury fury, MemoryBuffer buffer, Map map, int size) if (genericType.getTypeParametersCount() < 2) { Tuple2 kvGenericType = getKVGenericType(genericType); if (keyGenericType == objType && valueGenericType == objType) { - javaChunkRead(fury, buffer, map, size); + generalChunkRead(fury, buffer, map, size); return; } keyGenericType = kvGenericType.f0; @@ -647,7 +555,7 @@ private void genericJavaRead(Fury fury, MemoryBuffer buffer, Map map, int size) boolean keyGenericTypeFinal = keyGenericType.isMonomorphic(); boolean valueGenericTypeFinal = valueGenericType.isMonomorphic(); if (keyGenericTypeFinal && valueGenericTypeFinal) { - javaKVTypesFinalRead(fury, buffer, map, keyGenericType, valueGenericType, generics, size); + javaKVTypesFinalChunkRead(fury, buffer, map, keyGenericType, valueGenericType, generics, size); } else if (keyGenericTypeFinal) { javaKeyTypeFinalRead(fury, buffer, map, keyGenericType, valueGenericType, generics, size); } else if (valueGenericTypeFinal) { @@ -660,6 +568,36 @@ private void genericJavaRead(Fury fury, MemoryBuffer buffer, Map map, int size) } } + private void javaKVTypesFinalChunkRead(Fury fury, + MemoryBuffer buffer, + Map map, + GenericType keyGenericType, + GenericType valueGenericType, + Generics generics, + int size) { + Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); + Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); + MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); + while (size > 0) { + byte chunkSize = buffer.readByte(); + byte header = buffer.readByte(); + Preconditions.checkArgument(chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); + for (byte i = 0; i < chunkSize; i++) { + Object key; + Object value; + generics.pushGenericType(keyGenericType); + key = mapChunkWriter.readFinalKey(buffer, header, keySerializer); + generics.popGenericType(); + generics.pushGenericType(valueGenericType); + value = mapChunkWriter.readFinalValue(buffer, header, valueSerializer); + generics.popGenericType(); + map.put(key, value); + size--; + } + } + + } + private void javaKVTypesFinalRead( Fury fury, MemoryBuffer buffer, diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapChunkWriter.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapChunkWriter.java new file mode 100644 index 0000000000..1d9acc393c --- /dev/null +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapChunkWriter.java @@ -0,0 +1,529 @@ +package org.apache.fury.serializer.collection; + +import org.apache.fury.Fury; +import org.apache.fury.memory.MemoryBuffer; +import org.apache.fury.resolver.ClassInfo; +import org.apache.fury.resolver.ClassInfoHolder; +import org.apache.fury.resolver.ClassResolver; +import org.apache.fury.resolver.RefResolver; +import org.apache.fury.serializer.Serializer; +import org.apache.fury.type.GenericType; +import org.apache.fury.util.Preconditions; + +import java.util.Map; + +public class MapChunkWriter { + + private static final int MAX_CHUNK_SIZE = 127; + + public MapChunkWriter(Fury fury) { + this.fury = fury; + } + + private int header = 0; + private int startOffset; + private int chunkSize; + private Class keyClass = null; + private Class valueClass = null; + private final Fury fury; + private boolean writeKeyClassInfo = false; + private boolean writeValueClassInfo = false; + + /** + * mark chunk write finish + */ + private boolean markChunkWriteFinish = false; + /** + * preserve two byte for header and chunk size and record the write index + * so that we can write key value at first, write header and chunk size when the chunk is finish at correct position + */ + private boolean preserveByteForHeaderAndChunkSize = false; + + private void preserveByteForHeaderAndChunkSize(MemoryBuffer memoryBuffer) { + if (!preserveByteForHeaderAndChunkSize) { + return; + } + int writerIndex = memoryBuffer.writerIndex(); + // preserve two byte for header and chunk size + memoryBuffer.writerIndex(writerIndex + 2); + this.startOffset = writerIndex; + preserveByteForHeaderAndChunkSize = true; + } + + public void increaseChunkSize() { + chunkSize++; + } + + public void resetIfNeed(Object key, MemoryBuffer memoryBuffer) { + // if chunkSize reach max chunk size or key is null, start a new chunk + if (chunkSize >= MAX_CHUNK_SIZE || key == null) { + reset(memoryBuffer); + } + } + + + public void generalChunkWrite(Object key, Object value, MemoryBuffer memoryBuffer, ClassResolver classResolver, RefResolver refResolver, ClassInfoHolder keyClassInfoWriteCache, ClassInfoHolder valueClassInfoWriteCache) { + if (!markChunkWriteFinish) { + writeKey(key, memoryBuffer, classResolver, refResolver, keyClassInfoWriteCache); + writeValue(value, memoryBuffer, classResolver, refResolver, valueClassInfoWriteCache); + increaseChunkSize(); + resetIfNeed(key, memoryBuffer); + } + writeJavaRefOptimized(fury, classResolver, refResolver, memoryBuffer, key, keyClassInfoWriteCache); + writeJavaRefOptimized(fury, classResolver, refResolver, memoryBuffer, value, valueClassInfoWriteCache); + } + + public void generalChunkRead(MemoryBuffer memoryBuffer, ClassResolver classResolver, RefResolver refResolver, Map map, int size, ClassInfoHolder keyClassInfoReadCache, ClassInfoHolder valueClassInfoReadCache) { + while(size > 0) { + Object key; + Object value; + if (!markChunkWriteFinish) { + byte headerSize = memoryBuffer.readByte(); + Preconditions.checkArgument(headerSize >= 0, "unexpected header size"); + if (headerSize == 0) { + key = fury.readRef(memoryBuffer, keyClassInfoReadCache); + value = fury.readRef(memoryBuffer, valueClassInfoReadCache); + markChunkWriteFinish = true; + } else { + this.header = memoryBuffer.readByte(); + Serializer keySerializer = null; + Serializer valueSerializer = null; + while (headerSize > 0) { + if (keyHasNull()) { + byte nullFlag = memoryBuffer.readByte(); + Preconditions.checkArgument(nullFlag == Fury.NULL_FLAG, "unexpected NULL_FLAG"); + key = null; + } else { + if (keySerializer == null) { + keySerializer = classResolver.readClassInfo(memoryBuffer, keyClassInfoReadCache).getSerializer(); + } + boolean trackingKeyRef = keySerializer.needToWriteRef(); + if (!trackingKeyRef) { + if (!keyIsNotSameType()) { + key = keySerializer.read(memoryBuffer); + } else { + key = fury.readNonRef(memoryBuffer, keyClassInfoReadCache);; + } + } else { + if (!keyIsNotSameType()) { + key = keySerializer.read(memoryBuffer); + } else { + key = fury.readRef(memoryBuffer, keyClassInfoReadCache); + } + } + } + if (valueHasNull()) { + + } else { + + } + headerSize--; + } + } + } else { + key = fury.readRef(memoryBuffer, keyClassInfoReadCache); + value = fury.readRef(memoryBuffer, valueClassInfoReadCache); + } + map.put(key, value); + size--; + } + + Object key = fury.readRef(memoryBuffer, keyClassInfoReadCache); + Object value = fury.readRef(memoryBuffer, valueClassInfoReadCache); + map.put(key, value); + } + + public void writeFinalKey(Object key, MemoryBuffer buffer, Serializer keySerializer) { + preserveByteForHeaderAndChunkSize(buffer); + boolean trackingKeyRef = keySerializer.needToWriteRef(); + if (!trackingKeyRef) { + // map key has one null at most, use one chunk to write + if (key == null) { + if (chunkSize > 0) { + reset(buffer); + } + header |= MapFlags.KEY_HAS_NULL; + buffer.writeByte(Fury.NULL_FLAG); + } else { + keySerializer.write(buffer, key); + } + } else { + updateFinalKeyHeader(key, true); + RefResolver refResolver = fury.getRefResolver(); + if (!refResolver.writeRefOrNull(buffer, key)) { + keySerializer.write(buffer, key); + } + } + } + + public Object readFinalKey(MemoryBuffer buffer, int header, Serializer keySerializer) { + if ((header & MapFlags.KEY_HAS_NULL) == MapFlags.KEY_HAS_NULL) { + byte nullFlag = buffer.readByte(); + Preconditions.checkArgument(nullFlag == Fury.NULL_FLAG, "unexpected NULL_FLAG"); + return null; + } else { + boolean trackingKeyRef = keySerializer.needToWriteRef(); + if (trackingKeyRef) { + return fury.readRef(buffer, keySerializer); + } else { + return keySerializer.read(buffer); + } + } + } + + public void writeFinalValue(Object value, MemoryBuffer buffer, Serializer valueSerializer) { + preserveByteForHeaderAndChunkSize(buffer); + boolean trackingValueRef = valueSerializer.needToWriteRef(); + if (!trackingValueRef) { + if (value == null) { + //if value has null before, no need to reset chunk + if (chunkSize > 0 && !valueHasNull()) { + reset(buffer); + } + header |= MapFlags.VALUE_HAS_NULL; + buffer.writeByte(Fury.NULL_FLAG); + } else { + if (valueHasNull()) { + buffer.writeByte(Fury.NOT_NULL_VALUE_FLAG); + valueSerializer.write(buffer, value); + } else { + valueSerializer.write(buffer, value); + } + } + } else { + updateFinalValueHeader(value, true); + RefResolver refResolver = fury.getRefResolver(); + if (!refResolver.writeRefOrNull(buffer, value)) { + valueSerializer.write(buffer, value); + } + } + } + + public Object readFinalValue(MemoryBuffer buffer, int header, Serializer valueSerializer) { + if ((header & MapFlags.VALUE_HAS_NULL) == MapFlags.VALUE_HAS_NULL) { + byte nullFlag = buffer.readByte(); + if (nullFlag == Fury.NULL_FLAG) { + return null; + } else { + return valueSerializer.read(buffer); + } + } else { + boolean trackingValueRef = valueSerializer.needToWriteRef(); + if (trackingValueRef) { + return fury.readRef(buffer, valueSerializer); + } else { + return valueSerializer.read(buffer); + } + } + } + + public void writeKeyWithGenericType(Object key, MemoryBuffer buffer, ClassResolver classResolver, RefResolver refResolver, GenericType keyGenericType, ClassInfoHolder keyClassInfoWriteCache) { + boolean trackingKeyRef = fury.getClassResolver().needToWriteRef(keyGenericType.getCls()); + preserveByteForHeaderAndChunkSize(buffer); + if (key == null) { + if ((header & MapFlags.KEY_HAS_NULL) != MapFlags.KEY_HAS_NULL) { + reset(buffer); + } + updateKeyHeader(null, buffer, keyClassInfoWriteCache, trackingKeyRef); + buffer.writeByte(Fury.NULL_FLAG); + } else { + writeKeyNonNull(key, buffer, classResolver, refResolver, keyClassInfoWriteCache, trackingKeyRef); + } + } + + private void writeKeyNonNull(Object key, MemoryBuffer buffer, ClassResolver classResolver, RefResolver refResolver, ClassInfoHolder keyClassInfoWriteCache, boolean trackingKeyRef) { + updateKeyHeader(key, buffer, keyClassInfoWriteCache, trackingKeyRef); + if (!trackingKeyRef) { + if (!keyIsNotSameType()) { + keyClassInfoWriteCache.getSerializer().write(buffer, key); + } else { + if (valueNotSameType()) { + markChunkWriteFinish(buffer); + } + writeJavaRefOptimized(fury, classResolver, refResolver, true, buffer, key, keyClassInfoWriteCache); + } + } else { + if (!keyIsNotSameType()) { + if (!refResolver.writeRefOrNull(buffer, key)) { + keyClassInfoWriteCache.getSerializer().write(buffer, key); + } + } else { + if (valueNotSameType()) { + markChunkWriteFinish(buffer); + } + writeJavaRefOptimized(fury, classResolver, refResolver, true, buffer, key, keyClassInfoWriteCache); + } + } + + } + + public void writeValueWithGenericType(Object value, MemoryBuffer buffer, ClassResolver classResolver, RefResolver refResolver, GenericType valueGenericType, ClassInfoHolder valueClassInfoWriteCache) { + boolean trackingValueRef = fury.getClassResolver().needToWriteRef(valueGenericType.getCls()); + preserveByteForHeaderAndChunkSize(buffer); + if (value == null) { + if (chunkSize > 0 && (header & MapFlags.VALUE_HAS_NULL) != MapFlags.VALUE_HAS_NULL) { + reset(buffer); + } + updateValueHeader(null, buffer, valueClassInfoWriteCache, trackingValueRef); + buffer.writeByte(Fury.NULL_FLAG); + } else { + writeValueNonNull(value, buffer, classResolver, refResolver, valueClassInfoWriteCache, trackingValueRef); + } + } + + + private void writeValueNonNull(Object value, MemoryBuffer buffer, ClassResolver classResolver, RefResolver refResolver, ClassInfoHolder valueClassInfoWriteCache, boolean trackingValueRef) { + updateValueHeader(value, buffer, valueClassInfoWriteCache, trackingValueRef); + if (!trackingValueRef) { + if (!valueNotSameType()) { + Serializer valueSerializer = valueClassInfoWriteCache.getSerializer(); + if (valueHasNull()) { + buffer.writeByte(Fury.NOT_NULL_VALUE_FLAG); + valueSerializer.write(buffer, value); + } else { + valueSerializer.write(buffer, value); + } + } else { + if (keyIsNotSameType()) { + markChunkWriteFinish(buffer); + } + writeJavaRefOptimized(fury, classResolver, refResolver, false, buffer, value, valueClassInfoWriteCache); + } + } else { + if (!valueNotSameType()) { + if (!refResolver.writeRefOrNull(buffer, value)) { + valueClassInfoWriteCache.getSerializer().write(buffer, value); + } + } else { + if (keyIsNotSameType()) { + markChunkWriteFinish(buffer); + } + writeJavaRefOptimized(fury, classResolver, refResolver, true, buffer, value, valueClassInfoWriteCache); + } + } + } + + + public void writeKey(Object key, MemoryBuffer buffer, ClassResolver classResolver, RefResolver refResolver, ClassInfoHolder keyClassInfoWriteCache) { + preserveByteForHeaderAndChunkSize(buffer); + if (key == null) { + //If chunk size > 0 means there are non null keys in the chunk, + // then the chunk needs to be reset to store the null. + // Otherwise, it means encountering null for the first time and not performing a reset operation + if (chunkSize > 0) { + reset(buffer); + } + header |= MapFlags.KEY_HAS_NULL; + buffer.writeByte(Fury.NULL_FLAG); + } else { + ClassInfo classInfo = classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache); + boolean trackingKeyRef = classInfo.getSerializer().needToWriteRef(); + writeKeyNonNull(key, buffer, classResolver, refResolver, keyClassInfoWriteCache, trackingKeyRef); + } + } + + + public void writeValue(Object value + , MemoryBuffer buffer + , ClassResolver classResolver + , RefResolver refResolver + , ClassInfoHolder valueClassInfoWriteCache) { + preserveByteForHeaderAndChunkSize(buffer); + if (value == null) { + if (chunkSize > 0 && !valueHasNull()) { + reset(buffer); + } + header |= MapFlags.VALUE_HAS_NULL; + buffer.writeByte(Fury.NULL_FLAG); + } else { + ClassInfo classInfo = classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache); + boolean trackingValueRef = classInfo.getSerializer().needToWriteRef(); + writeValueNonNull(value, buffer, classResolver, refResolver, valueClassInfoWriteCache, trackingValueRef); + } + } + + private void updateKeyHeader(Object key, MemoryBuffer memoryBuffer, ClassInfoHolder keyClassInfoWriteCache, boolean trackingKeyRef) { + if (key == null) { + header |= MapFlags.KEY_HAS_NULL; + } else { + if (keyClass == null) { + keyClass = key.getClass(); + } else if (keyClass != key.getClass()) { + header |= MapFlags.KEY_NOT_SAME_TYPE; + } + if (trackingKeyRef) { + header |= MapFlags.TRACKING_KEY_REF; + } + ClassResolver classResolver = fury.getClassResolver(); + ClassInfo classInfo = classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache); + if (!writeKeyClassInfo) { + classResolver.writeClass(memoryBuffer, classInfo); + writeKeyClassInfo = true; + } + } + + } + + private void updateFinalKeyHeader(Object key, boolean trackingKeyRef) { + if (trackingKeyRef) { + header |= MapFlags.TRACKING_KEY_REF; + } else { + if (key == null) { + header |= MapFlags.KEY_HAS_NULL; + } + } + } + + private void updateFinalValueHeader(Object value, boolean trackingValueRef) { + if (trackingValueRef) { + header |= MapFlags.TRACKING_VALUE_REF; + } else { + if (value == null) { + header |= MapFlags.VALUE_HAS_NULL; + } + } + } + + private void updateValueHeader(Object value, MemoryBuffer memoryBuffer, ClassInfoHolder valueClassInfoWriteCache, boolean trackingValueRef) { + if (value == null) { + header |= MapFlags.VALUE_HAS_NULL; + } else { + if (valueClass == null) { + valueClass = value.getClass(); + } else if (valueClass != value.getClass()) { + header |= MapFlags.VALUE_NOT_SAME_TYPE; + } + if (trackingValueRef) { + header |= MapFlags.TRACKING_VALUE_REF; + } + ClassResolver classResolver = fury.getClassResolver(); + ClassInfo classInfo = classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache); + if (!writeValueClassInfo) { + classResolver.writeClass(memoryBuffer, classInfo); + writeValueClassInfo = true; + } + } + } + + /** + * update chunk size and header, if chunk size == 0, do nothing + * + * @param memoryBuffer memoryBuffer which is written + */ + public void writeHeader(MemoryBuffer memoryBuffer) { + if (chunkSize > 0) { + int currentWriteIndex = memoryBuffer.writerIndex(); + memoryBuffer.writerIndex(startOffset); + memoryBuffer.writeByte(chunkSize); + memoryBuffer.writeByte(header); + memoryBuffer.writerIndex(currentWriteIndex); + chunkSize = 0; + } + } + + /** + * use chunk size = 0 to mark chunk write finish, + * if mark chunk write finish which means predict failed, chunk write is finish, + * rest of map will be written by generalJavaWrite + * + * @param memoryBuffer memoryBuffer which is written + */ + public void markChunkWriteFinish(MemoryBuffer memoryBuffer) { + if (!markChunkWriteFinish) { + writeHeader(memoryBuffer); + //set chunk size = 0 + memoryBuffer.writeByte(0); + markChunkWriteFinish = true; + } + } + + /** + * chunk size reach max size, start new chunk, no need reset keyClass and value Class + * + * @param memoryBuffer memoryBuffer which is written + */ + public void reset(MemoryBuffer memoryBuffer) { + writeHeader(memoryBuffer); + header = 0; + chunkSize = 0; + preserveByteForHeaderAndChunkSize = false; + } + + //todo 这两个方法的区别是什么 + private void writeJavaRefOptimized( + Fury fury, + ClassResolver classResolver, + RefResolver refResolver, + boolean trackingRef, + MemoryBuffer buffer, + Object obj, + ClassInfoHolder classInfoHolder) { + if (trackingRef) { + if (!refResolver.writeNullFlag(buffer, obj)) { + fury.writeRef(buffer, obj, classResolver.getClassInfo(obj.getClass(), classInfoHolder)); + } + } else { + if (obj == null) { + buffer.writeByte(Fury.NULL_FLAG); + } else { + buffer.writeByte(Fury.NOT_NULL_VALUE_FLAG); + fury.writeNonRef(buffer, obj, classResolver.getClassInfo(obj.getClass(), classInfoHolder)); + } + } + } + + private void writeJavaRefOptimized( + Fury fury, + ClassResolver classResolver, + RefResolver refResolver, + MemoryBuffer buffer, + Object obj, + ClassInfoHolder classInfoHolder) { + if (!refResolver.writeNullFlag(buffer, obj)) { + fury.writeRef(buffer, obj, classResolver.getClassInfo(obj.getClass(), classInfoHolder)); + } + } + + private Object readJavaRefOptimized( + Fury fury, + RefResolver refResolver, + boolean trackingRef, + MemoryBuffer buffer, + ClassInfoHolder classInfoHolder) { + if (trackingRef) { + int nextReadRefId = refResolver.tryPreserveRefId(buffer); + if (nextReadRefId >= Fury.NOT_NULL_VALUE_FLAG) { + Object obj = fury.readNonRef(buffer, classInfoHolder); + refResolver.setReadObject(nextReadRefId, obj); + return obj; + } else { + return refResolver.getReadObject(); + } + } else { + byte headFlag = buffer.readByte(); + if (headFlag == Fury.NULL_FLAG) { + return null; + } else { + return fury.readNonRef(buffer, classInfoHolder); + } + } + } + + + private boolean keyHasNull() { + return (header & MapFlags.KEY_HAS_NULL) == MapFlags.KEY_HAS_NULL; + } + + private boolean valueHasNull() { + return (header & MapFlags.VALUE_HAS_NULL) == MapFlags.VALUE_HAS_NULL; + } + + private boolean valueNotSameType() { + return (header & MapFlags.VALUE_NOT_SAME_TYPE) == MapFlags.VALUE_NOT_SAME_TYPE; + } + + private boolean keyIsNotSameType() { + return (header & MapFlags.KEY_NOT_SAME_TYPE) == MapFlags.KEY_NOT_SAME_TYPE; + } + +} diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapFlags.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapFlags.java index 3e76679b35..1c51a20a5c 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapFlags.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapFlags.java @@ -32,35 +32,35 @@ public class MapFlags { */ public static int KEY_HAS_NULL = 0b10; - /** - * Whether key is not declare type. - */ - public static int KEY_NOT_DECL_TYPE = 0b100; +// /** +// * Whether key is not declare type. +// */ +// public static int KEY_NOT_DECL_TYPE = 0b100; /** * Whether keys type are different. */ - public static int KEY_NOT_SAME_TYPE = 0b1000; + public static int KEY_NOT_SAME_TYPE = 0b100; /** * Whether track value ref. */ - public static int TRACKING_VALUE_REF = 0b10000; + public static int TRACKING_VALUE_REF = 0b1000; /** * Whether value has null. */ - public static int VALUE_HAS_NULL = 0b100000; + public static int VALUE_HAS_NULL = 0b10000; /** * Whether value is not declare type. */ - public static int VALUE_NOT_DECL_TYPE = 0b1000000; +// public static int VALUE_NOT_DECL_TYPE = 0b1000000; /** * Whether values type are different. */ - public static int VALUE_NOT_SAME_TYPE = 0b10000000; + public static int VALUE_NOT_SAME_TYPE = 0b100000; } diff --git a/java/fury-core/src/test/java/org/apache/fury/serializer/collection/MapSerializersTest.java b/java/fury-core/src/test/java/org/apache/fury/serializer/collection/MapSerializersTest.java index 6aa5f2cfaf..cd9f642934 100644 --- a/java/fury-core/src/test/java/org/apache/fury/serializer/collection/MapSerializersTest.java +++ b/java/fury-core/src/test/java/org/apache/fury/serializer/collection/MapSerializersTest.java @@ -137,6 +137,7 @@ public void testTreeMap() { .withLanguage(Language.JAVA) .withRefTracking(referenceTracking) .requireClassRegistration(false) + .withCodegen(false) .build(); TreeMap map = new TreeMap<>( @@ -151,7 +152,7 @@ public void testTreeMap() { }); map.put("str1", "1"); map.put("str2", "1"); - assertEquals(map, serDe(fury, map)); +// assertEquals(map, serDe(fury, map)); BeanForMap beanForMap = new BeanForMap(); assertEquals(beanForMap, serDe(fury, beanForMap)); } From 1e03081a0f05a0e30f3a445cff1198b1b88cd251 Mon Sep 17 00:00:00 2001 From: hening Date: Mon, 29 Jul 2024 17:50:12 +0800 Subject: [PATCH 07/18] update --- .../collection/AbstractMapSerializer.java | 301 +++++++--- .../serializer/collection/MapChunkWriter.java | 548 +++++++++++------- 2 files changed, 553 insertions(+), 296 deletions(-) diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java index 8f6bc05ad5..d563a7f9da 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java @@ -131,11 +131,11 @@ private void javaWriteWithKeySerializers(Map map, MemoryBuffer buffer, Serialize RefResolver refResolver = fury.getRefResolver(); for (Object object : map.entrySet()) { Map.Entry entry = (Map.Entry) object; + mapChunkWriter = mapChunkWriter.next(entry.getKey(), entry.getValue(), buffer); mapChunkWriter.writeFinalKey(entry.getKey(), buffer, keySerializer); Object value = entry.getValue(); - mapChunkWriter.writeValue(value, buffer, classResolver, refResolver, valueClassInfoWriteCache); + mapChunkWriter.writeValue(value, buffer, classResolver, refResolver, fury.trackingRef(), valueClassInfoWriteCache); mapChunkWriter.increaseChunkSize(); - mapChunkWriter.resetIfNeed(entry.getKey(), buffer); } mapChunkWriter.writeHeader(buffer); } @@ -147,10 +147,10 @@ private void javaWriteWithValueSerializers(Map map, MemoryBuffer buffer, Seriali for (Object object : map.entrySet()) { Map.Entry entry = (Map.Entry) object; Object key = entry.getKey(); - mapChunkWriter.writeKey(key, buffer, classResolver, refResolver, keyClassInfoWriteCache); + mapChunkWriter = mapChunkWriter.next(entry.getKey(), entry.getValue(), buffer); + mapChunkWriter.writeKey(key, buffer, classResolver, refResolver, fury.trackingRef(), keyClassInfoWriteCache); mapChunkWriter.writeFinalValue(entry.getValue(), buffer, valueSerializer); mapChunkWriter.increaseChunkSize(); - mapChunkWriter.resetIfNeed(key, buffer); } mapChunkWriter.writeHeader(buffer); } @@ -163,13 +163,13 @@ private void javaWriteWithKVSerializers( Serializer valueSerializer) { MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); for (Object object : map.entrySet()) { - Map.Entry entry = (Map.Entry) object; - Object key = entry.getKey(); - Object value = entry.getValue(); + Map.Entry entry = (Map.Entry) object; + Object key = entry.getKey(); + Object value = entry.getValue(); + mapChunkWriter = mapChunkWriter.next(entry.getKey(), entry.getValue(), buffer); mapChunkWriter.writeFinalValue(value, buffer, valueSerializer); mapChunkWriter.writeFinalKey(key, buffer, keySerializer); mapChunkWriter.increaseChunkSize(); - mapChunkWriter.resetIfNeed(key, buffer); // fury.writeRef(buffer, key, keySerializer); // fury.writeRef(buffer, value, valueSerializer); } @@ -180,7 +180,7 @@ private void genericJavaWrite(Fury fury, MemoryBuffer buffer, Map map) { Generics generics = fury.getGenerics(); GenericType genericType = generics.nextGenericType(); if (genericType == null) { - javaChunkWrite(fury, buffer, map); + generalJavaChunkWrite(fury, buffer, map); } else { GenericType keyGenericType = genericType.getTypeParameter0(); GenericType valueGenericType = genericType.getTypeParameter1(); @@ -190,7 +190,7 @@ private void genericJavaWrite(Fury fury, MemoryBuffer buffer, Map map) { if (genericType.getTypeParametersCount() < 2) { Tuple2 kvGenericType = getKVGenericType(genericType); if (keyGenericType == objType && valueGenericType == objType) { - javaChunkWrite(fury, buffer, map); + generalJavaChunkWrite(fury, buffer, map); return; } keyGenericType = kvGenericType.f0; @@ -267,6 +267,7 @@ private void javaKVTypesFinalChunkWrite( Map.Entry entry = (Map.Entry) object; Object key = entry.getKey(); Object value = entry.getValue(); + mapChunkWriter = mapChunkWriter.next(key, value, buffer); generics.pushGenericType(keyGenericType); mapChunkWriter.writeFinalKey(key, buffer, keySerializer); generics.popGenericType(); @@ -274,7 +275,6 @@ private void javaKVTypesFinalChunkWrite( mapChunkWriter.writeFinalValue(value, buffer, valueSerializer); generics.popGenericType(); mapChunkWriter.increaseChunkSize(); - mapChunkWriter.resetIfNeed(key, buffer); } mapChunkWriter.writeHeader(buffer); } @@ -319,18 +319,19 @@ private void javaKeyTypeFinalChunkWrite( ClassResolver classResolver = fury.getClassResolver(); RefResolver refResolver = fury.getRefResolver(); Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); + boolean trackingValueRef = fury.getClassResolver().needToWriteRef(valueGenericType.getCls()); for (Object object : map.entrySet()) { Map.Entry entry = (Map.Entry) object; Object key = entry.getKey(); Object value = entry.getValue(); + mapChunkWriter = mapChunkWriter.next(key, value, buffer); generics.pushGenericType(keyGenericType); mapChunkWriter.writeFinalKey(key, buffer, keySerializer); generics.popGenericType(); generics.pushGenericType(valueGenericType); - mapChunkWriter.writeValueWithGenericType(value, buffer, classResolver, refResolver, valueGenericType, valueClassInfoWriteCache); + mapChunkWriter.writeValue(value, buffer, classResolver, refResolver, trackingValueRef, valueClassInfoWriteCache); generics.popGenericType(); mapChunkWriter.increaseChunkSize(); - mapChunkWriter.resetIfNeed(key, buffer); } mapChunkWriter.writeHeader(buffer); } @@ -342,22 +343,23 @@ private void javaValueTypeFinalChunkWrite( GenericType keyGenericType, GenericType valueGenericType, Generics generics) { - final MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); + MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); ClassResolver classResolver = fury.getClassResolver(); RefResolver refResolver = fury.getRefResolver(); Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); + boolean trackingKeyRef = fury.getClassResolver().needToWriteRef(keyGenericType.getCls()); for (Object object : map.entrySet()) { Map.Entry entry = (Map.Entry) object; Object key = entry.getKey(); Object value = entry.getValue(); + mapChunkWriter = mapChunkWriter.next(key, value, buffer); generics.pushGenericType(keyGenericType); - mapChunkWriter.writeKeyWithGenericType(key, buffer, classResolver, refResolver, keyGenericType, keyClassInfoWriteCache); + mapChunkWriter.writeKey(key, buffer, classResolver, refResolver, trackingKeyRef, keyClassInfoWriteCache); generics.popGenericType(); generics.pushGenericType(valueGenericType); mapChunkWriter.writeFinalValue(value, buffer, valueSerializer); generics.popGenericType(); mapChunkWriter.increaseChunkSize(); - mapChunkWriter.resetIfNeed(key, buffer); } mapChunkWriter.writeHeader(buffer); } @@ -372,19 +374,25 @@ private void javaKVTypesNonFinalChunkWrite( MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); ClassResolver classResolver = fury.getClassResolver(); RefResolver refResolver = fury.getRefResolver(); + boolean trackingKeyRef = fury.getClassResolver().needToWriteRef(keyGenericType.getCls()); boolean trackingValueRef = fury.getClassResolver().needToWriteRef(valueGenericType.getCls()); for (Object object : map.entrySet()) { Map.Entry entry = (Map.Entry) object; Object key = entry.getKey(); Object value = entry.getValue(); - generics.pushGenericType(keyGenericType); - mapChunkWriter.writeKeyWithGenericType(key, buffer, classResolver, refResolver, keyGenericType, keyClassInfoWriteCache); - generics.popGenericType(); - generics.pushGenericType(valueGenericType); - mapChunkWriter.writeValue(value, buffer, classResolver, refResolver, valueClassInfoWriteCache); - generics.popGenericType(); - mapChunkWriter.increaseChunkSize(); - mapChunkWriter.resetIfNeed(entry.getKey(), buffer); + mapChunkWriter = mapChunkWriter.next(key, value, buffer); + if (mapChunkWriter.isMarkChunkWriteFinish()) { + writeJavaRefOptimized(fury, classResolver, refResolver, trackingKeyRef, buffer, key, keyClassInfoWriteCache); + writeJavaRefOptimized(fury, classResolver, refResolver, trackingValueRef, buffer, value, keyClassInfoWriteCache); + } else { + generics.pushGenericType(keyGenericType); + mapChunkWriter.writeKey(key, buffer, classResolver, refResolver, trackingKeyRef, keyClassInfoWriteCache); + generics.popGenericType(); + generics.pushGenericType(valueGenericType); + mapChunkWriter.writeValue(value, buffer, classResolver, refResolver, trackingValueRef, valueClassInfoWriteCache); + generics.popGenericType(); + mapChunkWriter.increaseChunkSize(); + } } mapChunkWriter.writeHeader(buffer); } @@ -401,7 +409,7 @@ private void generalJavaWrite(Fury fury, MemoryBuffer buffer, Map map) { } } - protected void javaChunkWrite(Fury fury, MemoryBuffer buffer, Map map) { + protected void generalJavaChunkWrite(Fury fury, MemoryBuffer buffer, Map map) { MapChunkWriter chunkWriter = new MapChunkWriter(fury); ClassResolver classResolver = fury.getClassResolver(); RefResolver refResolver = fury.getRefResolver(); @@ -409,7 +417,15 @@ protected void javaChunkWrite(Fury fury, MemoryBuffer buffer, Map map) { Map.Entry entry = (Map.Entry) object; Object key = entry.getKey(); Object value = entry.getValue(); - chunkWriter.generalChunkWrite(key, value, buffer, classResolver, refResolver, keyClassInfoWriteCache, valueClassInfoWriteCache); + chunkWriter = chunkWriter.next(key, value, buffer); + if (!chunkWriter.isMarkChunkWriteFinish()) { + chunkWriter.generalChunkWrite(key, value, buffer, classResolver, refResolver, keyClassInfoWriteCache, valueClassInfoWriteCache); + } else { + writeJavaRefOptimized( + fury, classResolver, refResolver, buffer, entry.getKey(), keyClassInfoWriteCache); + writeJavaRefOptimized( + fury, classResolver, refResolver, buffer, entry.getValue(), valueClassInfoWriteCache); + } } } @@ -503,21 +519,48 @@ protected final void readElements(MemoryBuffer buffer, int size, Map map) { if (keySerializer != null && valueSerializer != null) { javaChunkReadWithKVSerializers(buffer, map, size, keySerializer, valueSerializer); } else if (keySerializer != null) { - for (int i = 0; i < size; i++) { - Object key = fury.readRef(buffer, keySerializer); - map.put(key, fury.readRef(buffer, keyClassInfoReadCache)); - } + javaChunkReadWithKeySerializer(buffer, map, size, keySerializer); } else if (valueSerializer != null) { - for (int i = 0; i < size; i++) { - Object key = fury.readRef(buffer); - Object value = fury.readRef(buffer, valueSerializer); - map.put(key, value); - } + javaChunkReadWithValueSerializer(buffer, map, size, valueSerializer); } else { genericJavaRead(fury, buffer, map, size); } } + private void javaChunkReadWithKeySerializer(MemoryBuffer buffer, Map map, int size, Serializer keySerializer) { + MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); + while (size > 0) { + byte chunkSize = buffer.readByte(); + byte header = buffer.readByte(); + Preconditions.checkArgument(chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); + for (byte i = 0; i < chunkSize; i++) { + Object key; + Object value; + key = mapChunkWriter.readFinalKey(buffer, header, keySerializer); + value = mapChunkWriter.readValue(header, buffer, fury.getClassResolver(), fury.trackingRef(), valueClassInfoReadCache); + map.put(key, value); + size--; + } + } + } + + private void javaChunkReadWithValueSerializer(MemoryBuffer buffer, Map map, int size, Serializer valueSerializer) { + MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); + while (size > 0) { + byte chunkSize = buffer.readByte(); + byte header = buffer.readByte(); + Preconditions.checkArgument(chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); + for (byte i = 0; i < chunkSize; i++) { + Object key; + Object value; + key = mapChunkWriter.readKey(header, buffer, fury.getClassResolver(), fury.trackingRef(), keyClassInfoReadCache); + value = mapChunkWriter.readFinalValue(buffer, header, valueSerializer); + map.put(key, value); + size--; + } + } + } + private void javaChunkReadWithKVSerializers(MemoryBuffer buffer, Map map, int size, Serializer keySerializer, Serializer valueSerializer) { MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); while (size > 0) { @@ -546,7 +589,7 @@ private void genericJavaRead(Fury fury, MemoryBuffer buffer, Map map, int size) if (genericType.getTypeParametersCount() < 2) { Tuple2 kvGenericType = getKVGenericType(genericType); if (keyGenericType == objType && valueGenericType == objType) { - generalChunkRead(fury, buffer, map, size); + javaChunkRead(fury, buffer, map, size); return; } keyGenericType = kvGenericType.f0; @@ -557,11 +600,11 @@ private void genericJavaRead(Fury fury, MemoryBuffer buffer, Map map, int size) if (keyGenericTypeFinal && valueGenericTypeFinal) { javaKVTypesFinalChunkRead(fury, buffer, map, keyGenericType, valueGenericType, generics, size); } else if (keyGenericTypeFinal) { - javaKeyTypeFinalRead(fury, buffer, map, keyGenericType, valueGenericType, generics, size); + javaKeyTypeFinalChunkRead(fury, buffer, map, keyGenericType, valueGenericType, generics, size); } else if (valueGenericTypeFinal) { - javaValueTypeFinalRead(fury, buffer, map, keyGenericType, valueGenericType, generics, size); + javaValueTypeFinalChunkRead(fury, buffer, map, keyGenericType, valueGenericType, generics, size); } else { - javaKVTypesNonFinalRead( + javaKVTypesNonFinalChunkRead( fury, buffer, map, keyGenericType, valueGenericType, generics, size); } generics.popGenericType(); @@ -619,14 +662,43 @@ private void javaKVTypesFinalRead( } } - private void javaKeyTypeFinalRead( - Fury fury, - MemoryBuffer buffer, - Map map, - GenericType keyGenericType, - GenericType valueGenericType, - Generics generics, - int size) { + private void javaKeyTypeFinalChunkRead(Fury fury, + MemoryBuffer buffer, + Map map, + GenericType keyGenericType, + GenericType valueGenericType, + Generics generics, + int size) { + final MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); + ClassResolver classResolver = fury.getClassResolver(); + boolean trackingValueRef = classResolver.needToWriteRef(valueGenericType.getCls()); + Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); + while (size > 0) { + byte chunkSize = buffer.readByte(); + Preconditions.checkArgument(chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); + byte header = buffer.readByte(); + valueClassInfoReadCache.classInfo = classResolver.nilClassInfo(); + while(chunkSize > 0) { + generics.pushGenericType(keyGenericType); + Object key = mapChunkWriter.readFinalKey(buffer, header, keySerializer); + generics.pushGenericType(valueGenericType); + Object value = mapChunkWriter.readValue(header, buffer, classResolver, trackingValueRef, valueClassInfoReadCache); + generics.pushGenericType(valueGenericType); + generics.popGenericType(); + chunkSize--; + size--; + map.put(key, value); + } + } + } + + private void javaKeyTypeFinalRead(Fury fury + , MemoryBuffer buffer + , Map map + , GenericType keyGenericType + , GenericType valueGenericType + , Generics generics + , int size) { RefResolver refResolver = fury.getRefResolver(); boolean trackingValueRef = fury.getClassResolver().needToWriteRef(valueGenericType.getCls()); Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); @@ -643,6 +715,36 @@ private void javaKeyTypeFinalRead( } } + private void javaValueTypeFinalChunkRead(Fury fury + , MemoryBuffer buffer + , Map map + , GenericType keyGenericType + , GenericType valueGenericType + , Generics generics + , int size) { + MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); + boolean trackingKeyRef = fury.getClassResolver().needToWriteRef(keyGenericType.getCls()); + Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); + ClassResolver classResolver = fury.getClassResolver(); + while (size > 0) { + byte chunkSize = buffer.readByte(); + Preconditions.checkArgument(chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); + byte header = buffer.readByte(); + valueClassInfoReadCache.classInfo = classResolver.nilClassInfo(); + while(chunkSize > 0) { + generics.pushGenericType(keyGenericType); + Object key = mapChunkWriter.readKey(header, buffer, classResolver, trackingKeyRef, keyClassInfoReadCache); + generics.pushGenericType(valueGenericType); + Object value = mapChunkWriter.readFinalValue(buffer, header, valueSerializer); + generics.pushGenericType(valueGenericType); + generics.popGenericType(); + chunkSize--; + size--; + map.put(key, value); + } + } + } + private void javaValueTypeFinalRead( Fury fury, MemoryBuffer buffer, @@ -666,6 +768,49 @@ private void javaValueTypeFinalRead( } } + private void javaKVTypesNonFinalChunkRead(Fury fury, + MemoryBuffer buffer, + Map map, + GenericType keyGenericType, + GenericType valueGenericType, + Generics generics, + int size) { + final MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); + ClassResolver classResolver = fury.getClassResolver(); + RefResolver refResolver = fury.getRefResolver(); + boolean trackingKeyRef = classResolver.needToWriteRef(keyGenericType.getCls()); + boolean trackingValueRef = classResolver.needToWriteRef(valueGenericType.getCls()); + while (size > 0) { + byte chunkSize = buffer.readByte(); + Preconditions.checkArgument(chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); + if (chunkSize == 0) { + generics.pushGenericType(keyGenericType); + Object key = readJavaRefOptimized(fury, refResolver, trackingKeyRef, buffer, keyClassInfoReadCache); + generics.popGenericType(); + generics.pushGenericType(valueGenericType); + Object value = readJavaRefOptimized(fury, refResolver, trackingValueRef, buffer, valueClassInfoReadCache); + generics.popGenericType(); + map.put(key, value); + size--; + } else { + byte header = buffer.readByte(); + keyClassInfoReadCache.classInfo = classResolver.nilClassInfo(); + valueClassInfoReadCache.classInfo = classResolver.nilClassInfo(); + while(chunkSize > 0) { + generics.pushGenericType(keyGenericType); + Object key = mapChunkWriter.readKey(header, buffer, classResolver, trackingKeyRef, keyClassInfoReadCache); + generics.popGenericType(); + generics.pushGenericType(valueGenericType); + Object value = mapChunkWriter.readValue(header, buffer, classResolver, trackingValueRef, valueClassInfoReadCache); + generics.popGenericType(); + chunkSize--; + size--; + map.put(key, value); + } + } + } + } + private void javaKVTypesNonFinalRead( Fury fury, MemoryBuffer buffer, @@ -694,52 +839,30 @@ private void javaKVTypesNonFinalRead( private void javaChunkRead(Fury fury, MemoryBuffer buffer, Map map, int size) { - int copySize = size; - Serializer keySerializer = null; - Serializer valueSerializer = null; - byte chunkSize = buffer.readByte(); - Preconditions.checkArgument(chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); - if (chunkSize == 0) { - generalJavaRead(fury, buffer, map, size); - return; - } - byte header = buffer.readByte(); - while (copySize > 0 && chunkSize > 0) { - Object key; - Object value = null; - if (keySerializer == null) { - keySerializer = fury.getClassResolver().readClassInfo(buffer, keyClassInfoReadCache).getSerializer(); - } - if (valueSerializer == null) { - valueSerializer = fury.getClassResolver().readClassInfo(buffer, valueClassInfoReadCache).getSerializer(); - } - if ((header & MapFlags.TRACKING_KEY_REF) == MapFlags.TRACKING_KEY_REF) { - key = fury.readRef(buffer, keySerializer); - } else { - key = keySerializer.read(buffer); - } - if ((header & MapFlags.TRACKING_VALUE_REF) == MapFlags.TRACKING_VALUE_REF) { - value = fury.readRef(buffer, valueSerializer); + final MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); + ClassResolver classResolver = fury.getClassResolver(); + boolean trackingKeyRef = fury.trackingRef(); + while (size > 0) { + byte chunkSize = buffer.readByte(); + Preconditions.checkArgument(chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); + if (chunkSize == 0) { + Object key = fury.readRef(buffer, keyClassInfoReadCache); + Object value = fury.readRef(buffer, keyClassInfoReadCache); + map.put(key, value); + size--; } else { - if (buffer.readByte() != Fury.NULL_FLAG) { - value = valueSerializer.read(buffer); - } - } - //todo 1. object 2. fury.incDepth(1); - copySize--; - chunkSize--; - map.put(key, value); - if (chunkSize == 0 && copySize != 0) { - chunkSize = buffer.readByte(); - Preconditions.checkArgument(chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); - if (chunkSize == 0) { - generalJavaRead(fury, buffer, map, copySize); - } else { - header = buffer.readByte(); + byte header = buffer.readByte(); + keyClassInfoReadCache.classInfo = classResolver.nilClassInfo(); + valueClassInfoReadCache.classInfo = classResolver.nilClassInfo(); + while(chunkSize > 0) { + Object key = mapChunkWriter.readKey(header, buffer, classResolver, trackingKeyRef, keyClassInfoReadCache); + Object value = mapChunkWriter.readValue(header, buffer, classResolver, trackingKeyRef, valueClassInfoReadCache); + chunkSize--; + size--; + map.put(key, value); } } } - } private void generalJavaRead(Fury fury, MemoryBuffer buffer, Map map, int size) { diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapChunkWriter.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapChunkWriter.java index 1d9acc393c..5b69638be6 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapChunkWriter.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapChunkWriter.java @@ -7,11 +7,13 @@ import org.apache.fury.resolver.ClassResolver; import org.apache.fury.resolver.RefResolver; import org.apache.fury.serializer.Serializer; -import org.apache.fury.type.GenericType; import org.apache.fury.util.Preconditions; import java.util.Map; +/** + * todo 1 value如果有空的情况，value可能有多个空，写class信息，在哪里写，2 写class当前可能写重复了 + */ public class MapChunkWriter { private static final int MAX_CHUNK_SIZE = 127; @@ -28,6 +30,9 @@ public MapChunkWriter(Fury fury) { private final Fury fury; private boolean writeKeyClassInfo = false; private boolean writeValueClassInfo = false; + private boolean keyIsNotSameType = false; + private boolean valueIsNotSameType = false; + private boolean prevKeyIsNull = false; /** * mark chunk write finish @@ -50,87 +55,57 @@ private void preserveByteForHeaderAndChunkSize(MemoryBuffer memoryBuffer) { preserveByteForHeaderAndChunkSize = true; } + public MapChunkWriter next(Object key, Object value, MemoryBuffer buffer) { + if (!markChunkWriteFinish) { + if (key == null && chunkSize > 0) { + prevKeyIsNull = true; + reset(buffer); + } + if (prevKeyIsNull && key != null) { + reset(buffer); + } + if (value == null && chunkSize > 0 && !valueHasNull()) { + //if value has null before, no need to reset chunk + reset(buffer); + } + if (!keyIsNotSameType) { + this.keyIsNotSameType = judgeKeyIsNotSameType(key); + if (keyIsNotSameType) { + if (valueIsNotSameType) { + markChunkWriteFinish(buffer); + } else { + reset(buffer); + } + } + } + if (!valueIsNotSameType) { + this.valueIsNotSameType = judgeValueIsNotSameType(value); + if (valueIsNotSameType) { + if (keyIsNotSameType) { + markChunkWriteFinish(buffer); + } else { + reset(buffer); + } + } + } + if (chunkSize >= MAX_CHUNK_SIZE) { + reset(buffer); + } + } + return this; + } + public void increaseChunkSize() { chunkSize++; } - public void resetIfNeed(Object key, MemoryBuffer memoryBuffer) { - // if chunkSize reach max chunk size or key is null, start a new chunk - if (chunkSize >= MAX_CHUNK_SIZE || key == null) { - reset(memoryBuffer); - } - } public void generalChunkWrite(Object key, Object value, MemoryBuffer memoryBuffer, ClassResolver classResolver, RefResolver refResolver, ClassInfoHolder keyClassInfoWriteCache, ClassInfoHolder valueClassInfoWriteCache) { - if (!markChunkWriteFinish) { - writeKey(key, memoryBuffer, classResolver, refResolver, keyClassInfoWriteCache); - writeValue(value, memoryBuffer, classResolver, refResolver, valueClassInfoWriteCache); - increaseChunkSize(); - resetIfNeed(key, memoryBuffer); - } - writeJavaRefOptimized(fury, classResolver, refResolver, memoryBuffer, key, keyClassInfoWriteCache); - writeJavaRefOptimized(fury, classResolver, refResolver, memoryBuffer, value, valueClassInfoWriteCache); - } - - public void generalChunkRead(MemoryBuffer memoryBuffer, ClassResolver classResolver, RefResolver refResolver, Map map, int size, ClassInfoHolder keyClassInfoReadCache, ClassInfoHolder valueClassInfoReadCache) { - while(size > 0) { - Object key; - Object value; - if (!markChunkWriteFinish) { - byte headerSize = memoryBuffer.readByte(); - Preconditions.checkArgument(headerSize >= 0, "unexpected header size"); - if (headerSize == 0) { - key = fury.readRef(memoryBuffer, keyClassInfoReadCache); - value = fury.readRef(memoryBuffer, valueClassInfoReadCache); - markChunkWriteFinish = true; - } else { - this.header = memoryBuffer.readByte(); - Serializer keySerializer = null; - Serializer valueSerializer = null; - while (headerSize > 0) { - if (keyHasNull()) { - byte nullFlag = memoryBuffer.readByte(); - Preconditions.checkArgument(nullFlag == Fury.NULL_FLAG, "unexpected NULL_FLAG"); - key = null; - } else { - if (keySerializer == null) { - keySerializer = classResolver.readClassInfo(memoryBuffer, keyClassInfoReadCache).getSerializer(); - } - boolean trackingKeyRef = keySerializer.needToWriteRef(); - if (!trackingKeyRef) { - if (!keyIsNotSameType()) { - key = keySerializer.read(memoryBuffer); - } else { - key = fury.readNonRef(memoryBuffer, keyClassInfoReadCache);; - } - } else { - if (!keyIsNotSameType()) { - key = keySerializer.read(memoryBuffer); - } else { - key = fury.readRef(memoryBuffer, keyClassInfoReadCache); - } - } - } - if (valueHasNull()) { - - } else { - - } - headerSize--; - } - } - } else { - key = fury.readRef(memoryBuffer, keyClassInfoReadCache); - value = fury.readRef(memoryBuffer, valueClassInfoReadCache); - } - map.put(key, value); - size--; - } - - Object key = fury.readRef(memoryBuffer, keyClassInfoReadCache); - Object value = fury.readRef(memoryBuffer, valueClassInfoReadCache); - map.put(key, value); + final boolean trackingRef = fury.trackingRef(); + writeKey(key, memoryBuffer, classResolver, refResolver, trackingRef, keyClassInfoWriteCache); + writeValue(value, memoryBuffer, classResolver, refResolver, trackingRef, valueClassInfoWriteCache); + increaseChunkSize(); } public void writeFinalKey(Object key, MemoryBuffer buffer, Serializer keySerializer) { @@ -139,9 +114,6 @@ public void writeFinalKey(Object key, MemoryBuffer buffer, Serializer keySeriali if (!trackingKeyRef) { // map key has one null at most, use one chunk to write if (key == null) { - if (chunkSize > 0) { - reset(buffer); - } header |= MapFlags.KEY_HAS_NULL; buffer.writeByte(Fury.NULL_FLAG); } else { @@ -157,17 +129,18 @@ public void writeFinalKey(Object key, MemoryBuffer buffer, Serializer keySeriali } public Object readFinalKey(MemoryBuffer buffer, int header, Serializer keySerializer) { - if ((header & MapFlags.KEY_HAS_NULL) == MapFlags.KEY_HAS_NULL) { - byte nullFlag = buffer.readByte(); - Preconditions.checkArgument(nullFlag == Fury.NULL_FLAG, "unexpected NULL_FLAG"); - return null; - } else { - boolean trackingKeyRef = keySerializer.needToWriteRef(); - if (trackingKeyRef) { - return fury.readRef(buffer, keySerializer); + this.header = header; + boolean trackingKeyRef = keySerializer.needToWriteRef(); + if (!trackingKeyRef) { + if (keyHasNull()) { + byte nullFlag = buffer.readByte(); + Preconditions.checkArgument(nullFlag == Fury.NULL_FLAG, "unexpected NULL_FLAG"); + return null; } else { return keySerializer.read(buffer); } + } else { + return fury.readRef(buffer, keySerializer); } } @@ -176,10 +149,6 @@ public void writeFinalValue(Object value, MemoryBuffer buffer, Serializer valueS boolean trackingValueRef = valueSerializer.needToWriteRef(); if (!trackingValueRef) { if (value == null) { - //if value has null before, no need to reset chunk - if (chunkSize > 0 && !valueHasNull()) { - reset(buffer); - } header |= MapFlags.VALUE_HAS_NULL; buffer.writeByte(Fury.NULL_FLAG); } else { @@ -200,170 +169,333 @@ public void writeFinalValue(Object value, MemoryBuffer buffer, Serializer valueS } public Object readFinalValue(MemoryBuffer buffer, int header, Serializer valueSerializer) { - if ((header & MapFlags.VALUE_HAS_NULL) == MapFlags.VALUE_HAS_NULL) { - byte nullFlag = buffer.readByte(); - if (nullFlag == Fury.NULL_FLAG) { - return null; + this.header = header; + boolean trackingValueRef = valueSerializer.needToWriteRef(); + if (!trackingValueRef) { + if (valueHasNull()) { + byte flag = buffer.readByte(); + if (flag == Fury.NOT_NULL_VALUE_FLAG) { + return valueSerializer.read(buffer); + } else { + return null; + } } else { return valueSerializer.read(buffer); } } else { - boolean trackingValueRef = valueSerializer.needToWriteRef(); - if (trackingValueRef) { - return fury.readRef(buffer, valueSerializer); - } else { - return valueSerializer.read(buffer); - } + return fury.readRef(buffer, valueSerializer); } } - public void writeKeyWithGenericType(Object key, MemoryBuffer buffer, ClassResolver classResolver, RefResolver refResolver, GenericType keyGenericType, ClassInfoHolder keyClassInfoWriteCache) { - boolean trackingKeyRef = fury.getClassResolver().needToWriteRef(keyGenericType.getCls()); - preserveByteForHeaderAndChunkSize(buffer); + private boolean judgeKeyIsNotSameType(Object key) { if (key == null) { - if ((header & MapFlags.KEY_HAS_NULL) != MapFlags.KEY_HAS_NULL) { - reset(buffer); - } - updateKeyHeader(null, buffer, keyClassInfoWriteCache, trackingKeyRef); - buffer.writeByte(Fury.NULL_FLAG); - } else { - writeKeyNonNull(key, buffer, classResolver, refResolver, keyClassInfoWriteCache, trackingKeyRef); + return false; + } + if (keyClass == null) { + keyClass = key.getClass(); } + return keyClass != key.getClass(); } - private void writeKeyNonNull(Object key, MemoryBuffer buffer, ClassResolver classResolver, RefResolver refResolver, ClassInfoHolder keyClassInfoWriteCache, boolean trackingKeyRef) { - updateKeyHeader(key, buffer, keyClassInfoWriteCache, trackingKeyRef); + private boolean judgeValueIsNotSameType(Object value) { + if (value == null) { + return false; + } + if (valueClass == null) { + valueClass = value.getClass(); + } + return valueClass != value.getClass(); + } + + public void writeKey(Object key, MemoryBuffer buffer, ClassResolver classResolver, RefResolver refResolver, boolean trackingKeyRef, ClassInfoHolder keyClassInfoWriteCache) { + preserveByteForHeaderAndChunkSize(buffer); + //todo hening key == null提到外面？ if (!trackingKeyRef) { - if (!keyIsNotSameType()) { - keyClassInfoWriteCache.getSerializer().write(buffer, key); + if (key == null) { + updateKeyHeader(null, buffer, keyClassInfoWriteCache, false, false); + buffer.writeByte(Fury.NULL_FLAG); } else { - if (valueNotSameType()) { - markChunkWriteFinish(buffer); + if (!keyIsNotSameType) { + updateKeyHeader(key, buffer, keyClassInfoWriteCache, false, false); + keyClassInfoWriteCache.getSerializer().write(buffer, key); + } else { + updateKeyHeader(key, buffer, keyClassInfoWriteCache, false, true); + fury.writeNonRef(buffer, key, classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache)); } - writeJavaRefOptimized(fury, classResolver, refResolver, true, buffer, key, keyClassInfoWriteCache); } } else { - if (!keyIsNotSameType()) { - if (!refResolver.writeRefOrNull(buffer, key)) { - keyClassInfoWriteCache.getSerializer().write(buffer, key); - } + //todo 提到外面 + if (key == null) { + updateKeyHeader(null, buffer, keyClassInfoWriteCache, true, false); + buffer.writeByte(Fury.NULL_FLAG); } else { - if (valueNotSameType()) { - markChunkWriteFinish(buffer); + if (!keyIsNotSameType) { + updateKeyHeader(key, buffer, keyClassInfoWriteCache, true, false); + //todo key is not null, no need to write no null flag + fury.writeRef(buffer, key, keyClassInfoWriteCache.getSerializer()); + } else { + // todo hening remove write class + updateKeyHeader(key, buffer, keyClassInfoWriteCache, true, true); + if (!refResolver.writeNullFlag(buffer, key)) { + fury.writeRef(buffer, key, classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache)); + } } - writeJavaRefOptimized(fury, classResolver, refResolver, true, buffer, key, keyClassInfoWriteCache); } } } - public void writeValueWithGenericType(Object value, MemoryBuffer buffer, ClassResolver classResolver, RefResolver refResolver, GenericType valueGenericType, ClassInfoHolder valueClassInfoWriteCache) { - boolean trackingValueRef = fury.getClassResolver().needToWriteRef(valueGenericType.getCls()); - preserveByteForHeaderAndChunkSize(buffer); - if (value == null) { - if (chunkSize > 0 && (header & MapFlags.VALUE_HAS_NULL) != MapFlags.VALUE_HAS_NULL) { - reset(buffer); + public Object readKey(int header, MemoryBuffer memoryBuffer, ClassResolver classResolver, boolean trackingKeyRef, ClassInfoHolder keyClassInfoReadCache) { + this.header = header; + if (!trackingKeyRef) { + if (keyHasNull()) { + byte nullFlag = memoryBuffer.readByte(); + Preconditions.checkArgument(nullFlag != Fury.NULL_FLAG, "unexpected error"); + return null; + } else { + if (!keyIsNotSameType()) { + if (keyClassInfoReadCache.getSerializer() == null) { + classResolver.readClassInfo(memoryBuffer, keyClassInfoReadCache); + } + return keyClassInfoReadCache.getSerializer().read(memoryBuffer); + } else { + return fury.readNonRef(memoryBuffer, keyClassInfoReadCache); + } } - updateValueHeader(null, buffer, valueClassInfoWriteCache, trackingValueRef); - buffer.writeByte(Fury.NULL_FLAG); } else { - writeValueNonNull(value, buffer, classResolver, refResolver, valueClassInfoWriteCache, trackingValueRef); + if (keyHasNull()) { + byte nullFlag = memoryBuffer.readByte(); + Preconditions.checkArgument(nullFlag != Fury.NULL_FLAG, "unexpected error"); + return null; + } else { + if (!keyIsNotSameType()) { + if (keyClassInfoReadCache.getSerializer() == null) { + classResolver.readClassInfo(memoryBuffer, keyClassInfoReadCache); + } + return fury.readRef(memoryBuffer, keyClassInfoReadCache.getSerializer()); + } else { + return fury.readRef(memoryBuffer, keyClassInfoReadCache); + } + } } } - private void writeValueNonNull(Object value, MemoryBuffer buffer, ClassResolver classResolver, RefResolver refResolver, ClassInfoHolder valueClassInfoWriteCache, boolean trackingValueRef) { - updateValueHeader(value, buffer, valueClassInfoWriteCache, trackingValueRef); + public void writeValue(Object value, MemoryBuffer buffer, ClassResolver classResolver, RefResolver refResolver, boolean trackingValueRef, ClassInfoHolder valueClassInfoWriteCache) { + preserveByteForHeaderAndChunkSize(buffer); if (!trackingValueRef) { - if (!valueNotSameType()) { - Serializer valueSerializer = valueClassInfoWriteCache.getSerializer(); - if (valueHasNull()) { - buffer.writeByte(Fury.NOT_NULL_VALUE_FLAG); - valueSerializer.write(buffer, value); - } else { - valueSerializer.write(buffer, value); - } + if (value == null) { + updateValueHeader(null, buffer, valueClassInfoWriteCache, false, false); + buffer.writeByte(Fury.NULL_FLAG); } else { - if (keyIsNotSameType()) { - markChunkWriteFinish(buffer); + updateValueHeader(value, buffer, valueClassInfoWriteCache, false, valueIsNotSameType); + if (!valueIsNotSameType) { + writeValueClass(value, buffer, valueClassInfoWriteCache); + if (!valueHasNull()) { + valueClassInfoWriteCache.getSerializer().write(buffer, value); + } else { + buffer.writeByte(Fury.NOT_NULL_VALUE_FLAG); + valueClassInfoWriteCache.getSerializer().write(buffer, value); + } + } else { + fury.writeNonRef(buffer, value, classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache)); } - writeJavaRefOptimized(fury, classResolver, refResolver, false, buffer, value, valueClassInfoWriteCache); } } else { - if (!valueNotSameType()) { - if (!refResolver.writeRefOrNull(buffer, value)) { - valueClassInfoWriteCache.getSerializer().write(buffer, value); - } + if (value == null) { + updateValueHeader(null, buffer, valueClassInfoWriteCache, true, false); + buffer.writeByte(Fury.NULL_FLAG); } else { - if (keyIsNotSameType()) { - markChunkWriteFinish(buffer); + updateKeyHeader(value, buffer, valueClassInfoWriteCache, true, valueIsNotSameType); + if (!valueIsNotSameType) { + writeValueClass(value, buffer, valueClassInfoWriteCache); + if (!valueHasNull()) { + fury.writeRef(buffer, value, valueClassInfoWriteCache.getSerializer()); + } else { + buffer.writeByte(Fury.NOT_NULL_VALUE_FLAG); + fury.writeRef(buffer, value, valueClassInfoWriteCache.getSerializer()); + } + } else { + if (!refResolver.writeNullFlag(buffer, value)) { + fury.writeRef(buffer, value, classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache)); + } } - writeJavaRefOptimized(fury, classResolver, refResolver, true, buffer, value, valueClassInfoWriteCache); } } - } - - public void writeKey(Object key, MemoryBuffer buffer, ClassResolver classResolver, RefResolver refResolver, ClassInfoHolder keyClassInfoWriteCache) { - preserveByteForHeaderAndChunkSize(buffer); - if (key == null) { - //If chunk size > 0 means there are non null keys in the chunk, - // then the chunk needs to be reset to store the null. - // Otherwise, it means encountering null for the first time and not performing a reset operation - if (chunkSize > 0) { - reset(buffer); - } - header |= MapFlags.KEY_HAS_NULL; - buffer.writeByte(Fury.NULL_FLAG); - } else { - ClassInfo classInfo = classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache); - boolean trackingKeyRef = classInfo.getSerializer().needToWriteRef(); - writeKeyNonNull(key, buffer, classResolver, refResolver, keyClassInfoWriteCache, trackingKeyRef); - } } - - public void writeValue(Object value - , MemoryBuffer buffer - , ClassResolver classResolver - , RefResolver refResolver - , ClassInfoHolder valueClassInfoWriteCache) { - preserveByteForHeaderAndChunkSize(buffer); - if (value == null) { - if (chunkSize > 0 && !valueHasNull()) { - reset(buffer); + public Object readValue(int header, MemoryBuffer buffer, ClassResolver classResolver, boolean trackingValueRef, ClassInfoHolder valueClassInfoReadCache) { + this.header = header; + if (!trackingValueRef) { + if (valueHasNull()) { + byte flag = buffer.readByte(); + if (flag == Fury.NOT_NULL_VALUE_FLAG) { + if (valueClassInfoReadCache.getSerializer() == null) { + classResolver.readClassInfo(buffer, valueClassInfoReadCache); + } + return valueClassInfoReadCache.getSerializer().read(buffer); + } else { + return null; + } + } else { + return valueClassInfoReadCache.getSerializer().read(buffer); } - header |= MapFlags.VALUE_HAS_NULL; - buffer.writeByte(Fury.NULL_FLAG); } else { - ClassInfo classInfo = classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache); - boolean trackingValueRef = classInfo.getSerializer().needToWriteRef(); - writeValueNonNull(value, buffer, classResolver, refResolver, valueClassInfoWriteCache, trackingValueRef); + if (valueClassInfoReadCache.getSerializer() == null) { + classResolver.readClassInfo(buffer, valueClassInfoReadCache); + } + if (!valueIsNotSameType) { + if (valueHasNull()) { + byte flag = buffer.readByte(); + if (flag == Fury.NOT_NULL_VALUE_FLAG) { + if (valueClassInfoReadCache.getSerializer() == null) { + classResolver.readClassInfo(buffer, valueClassInfoReadCache); + } + return fury.readRef(buffer, valueClassInfoReadCache.getSerializer()); + } else { + return null; + } + } else { + return fury.readRef(buffer, valueClassInfoReadCache.getSerializer()); + } + } else { + return fury.readRef(buffer, valueClassInfoReadCache); + } } } - private void updateKeyHeader(Object key, MemoryBuffer memoryBuffer, ClassInfoHolder keyClassInfoWriteCache, boolean trackingKeyRef) { + +// private void writeValueNonNull(Object value, MemoryBuffer buffer, ClassResolver classResolver, RefResolver refResolver, ClassInfoHolder valueClassInfoWriteCache, boolean trackingValueRef) { +// updateValueHeader(value, buffer, valueClassInfoWriteCache, trackingValueRef); +// if (!trackingValueRef) { +// if (!valueNotSameType()) { +// Serializer valueSerializer = valueClassInfoWriteCache.getSerializer(); +// if (valueHasNull()) { +// buffer.writeByte(Fury.NOT_NULL_VALUE_FLAG); +// valueSerializer.write(buffer, value); +// } else { +// valueSerializer.write(buffer, value); +// } +// } else { +// if (keyIsNotSameType()) { +// markChunkWriteFinish(buffer); +// } +// writeJavaRefOptimized(fury, classResolver, refResolver, false, buffer, value, valueClassInfoWriteCache); +// } +// } else { +// if (!valueNotSameType()) { +// if (!refResolver.writeRefOrNull(buffer, value)) { +// valueClassInfoWriteCache.getSerializer().write(buffer, value); +// } +// } else { +// if (keyIsNotSameType()) { +// markChunkWriteFinish(buffer); +// } +// writeJavaRefOptimized(fury, classResolver, refResolver, true, buffer, value, valueClassInfoWriteCache); +// } +// } +// } + + +// public void writeKey(Object key, MemoryBuffer buffer, ClassResolver classResolver, RefResolver refResolver, ClassInfoHolder keyClassInfoWriteCache) { +// preserveByteForHeaderAndChunkSize(buffer); +// final boolean trackingRef = fury.trackingRef(); +// if (!trackingRef) { +// if (key == null) { +// header |= MapFlags.KEY_HAS_NULL; +// buffer.writeByte(Fury.NULL_FLAG); +// } else { +// updateKeyHeader(key, buffer, keyClassInfoWriteCache, false, keyIsNotSameType); +// if (!keyIsNotSameType) { +// keyClassInfoWriteCache.getSerializer().write(buffer, key); +// } else { +// fury.writeNonRef(buffer, key, classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache)); +// } +// } +// } else { +// if (key == null) { +// //todo remove writeClass +// updateKeyHeader(null, buffer, keyClassInfoWriteCache, true, false); +// buffer.writeByte(Fury.NULL_FLAG); +// } else { +// if (!keyIsNotSameType) { +// ClassInfo classInfo = classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache); +// boolean trackingKeyRef = classInfo.getSerializer().needToWriteRef(); +// updateKeyHeader(key, buffer, keyClassInfoWriteCache, trackingKeyRef, false); +// fury.writeRef(buffer, key, keyClassInfoWriteCache.getSerializer()); +// } else { +// writeJavaRefOptimized(fury, classResolver, refResolver, buffer, key, keyClassInfoWriteCache); +// } +// } +// } +// } +// +// +// public void writeValue(Object value +// , MemoryBuffer buffer +// , ClassResolver classResolver +// , RefResolver refResolver +// , ClassInfoHolder valueClassInfoWriteCache) { +// preserveByteForHeaderAndChunkSize(buffer); +// boolean trackingRef = fury.trackingRef(); +// if (!trackingRef) { +// if (value == null) { +// header |= MapFlags.VALUE_HAS_NULL; +// buffer.writeByte(Fury.NULL_FLAG); +// } else { +// updateValueHeader(value, buffer, valueClassInfoWriteCache, false, valueIsNotSameType); +// if (!valueIsNotSameType) { +// if (!valueHasNull()) { +// valueClassInfoWriteCache.getSerializer().write(buffer, value); +// } else { +// buffer.writeByte(Fury.NOT_NULL_VALUE_FLAG); +// valueClassInfoWriteCache.getSerializer().write(buffer, value); +// } +// } else { +// fury.writeNonRef(buffer, value, classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache)); +// } +// } +// } else { +// ClassInfo classInfo = classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache); +// boolean trackingValueRef = classInfo.getSerializer().needToWriteRef(); +// writeValueNonNull(value, buffer, classResolver, refResolver, valueClassInfoWriteCache, trackingValueRef); +// } +// +// } + + private void updateKeyHeader(Object key, MemoryBuffer memoryBuffer, ClassInfoHolder keyClassInfoWriteCache, boolean trackingKeyRef, boolean keyIsNotSameType) { if (key == null) { header |= MapFlags.KEY_HAS_NULL; } else { - if (keyClass == null) { - keyClass = key.getClass(); - } else if (keyClass != key.getClass()) { + if (keyIsNotSameType) { header |= MapFlags.KEY_NOT_SAME_TYPE; } if (trackingKeyRef) { header |= MapFlags.TRACKING_KEY_REF; } - ClassResolver classResolver = fury.getClassResolver(); - ClassInfo classInfo = classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache); - if (!writeKeyClassInfo) { - classResolver.writeClass(memoryBuffer, classInfo); - writeKeyClassInfo = true; - } } } + private void writeKeyClass(Object key, MemoryBuffer memoryBuffer, ClassInfoHolder keyClassInfoWriteCache) { + ClassResolver classResolver = fury.getClassResolver(); + ClassInfo classInfo = classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache); + if (!writeKeyClassInfo) { + classResolver.writeClass(memoryBuffer, classInfo); + writeKeyClassInfo = true; + } + } + + private void writeValueClass(Object value, MemoryBuffer memoryBuffer, ClassInfoHolder valueClassInfoWriteCache) { + ClassResolver classResolver = fury.getClassResolver(); + ClassInfo classInfo = classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache); + if (!writeValueClassInfo) { + classResolver.writeClass(memoryBuffer, classInfo); + writeValueClassInfo = true; + } + } + private void updateFinalKeyHeader(Object key, boolean trackingKeyRef) { if (trackingKeyRef) { header |= MapFlags.TRACKING_KEY_REF; @@ -384,24 +516,16 @@ private void updateFinalValueHeader(Object value, boolean trackingValueRef) { } } - private void updateValueHeader(Object value, MemoryBuffer memoryBuffer, ClassInfoHolder valueClassInfoWriteCache, boolean trackingValueRef) { + private void updateValueHeader(Object value, MemoryBuffer memoryBuffer, ClassInfoHolder valueClassInfoWriteCache, boolean trackingValueRef, boolean valueIsNotSameType) { if (value == null) { header |= MapFlags.VALUE_HAS_NULL; } else { - if (valueClass == null) { - valueClass = value.getClass(); - } else if (valueClass != value.getClass()) { + if (valueIsNotSameType) { header |= MapFlags.VALUE_NOT_SAME_TYPE; } if (trackingValueRef) { header |= MapFlags.TRACKING_VALUE_REF; } - ClassResolver classResolver = fury.getClassResolver(); - ClassInfo classInfo = classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache); - if (!writeValueClassInfo) { - classResolver.writeClass(memoryBuffer, classInfo); - writeValueClassInfo = true; - } } } @@ -447,9 +571,13 @@ public void reset(MemoryBuffer memoryBuffer) { header = 0; chunkSize = 0; preserveByteForHeaderAndChunkSize = false; + writeKeyClassInfo = false; + writeValueClassInfo = false; + prevKeyIsNull = false; + keyClass = null; + valueClass = null; } - //todo 这两个方法的区别是什么 private void writeJavaRefOptimized( Fury fury, ClassResolver classResolver, @@ -526,4 +654,10 @@ private boolean keyIsNotSameType() { return (header & MapFlags.KEY_NOT_SAME_TYPE) == MapFlags.KEY_NOT_SAME_TYPE; } + public boolean isMarkChunkWriteFinish() { + return markChunkWriteFinish; + } + + + } From b382f1dad6b636d7c8837b62c2c11e6f4f99a7fb Mon Sep 17 00:00:00 2001 From: hening Date: Mon, 29 Jul 2024 21:04:41 +0800 Subject: [PATCH 08/18] update --- .../src/main/java/org/apache/fury/Fury.java | 9 ++ .../collection/AbstractMapSerializer.java | 3 +- .../serializer/collection/MapChunkWriter.java | 90 ++++++++++--------- 3 files changed, 59 insertions(+), 43 deletions(-) diff --git a/java/fury-core/src/main/java/org/apache/fury/Fury.java b/java/fury-core/src/main/java/org/apache/fury/Fury.java index ccef2aaa3d..62fb87a0f8 100644 --- a/java/fury-core/src/main/java/org/apache/fury/Fury.java +++ b/java/fury-core/src/main/java/org/apache/fury/Fury.java @@ -884,6 +884,15 @@ public Object readNullable(MemoryBuffer buffer) { } } + public Object readNullable(MemoryBuffer buffer, ClassInfoHolder classInfoHolder) { + byte headFlag = buffer.readByte(); + if (headFlag == Fury.NULL_FLAG) { + return null; + } else { + return readNonRef(buffer, classInfoHolder); + } + } + /** Class should be read already. */ public Object readData(MemoryBuffer buffer, ClassInfo classInfo) { depth++; diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java index d563a7f9da..55777b5f4f 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java @@ -427,6 +427,7 @@ protected void generalJavaChunkWrite(Fury fury, MemoryBuffer buffer, Map map) { fury, classResolver, refResolver, buffer, entry.getValue(), valueClassInfoWriteCache); } } + chunkWriter.writeHeader(buffer); } @@ -523,7 +524,7 @@ protected final void readElements(MemoryBuffer buffer, int size, Map map) { } else if (valueSerializer != null) { javaChunkReadWithValueSerializer(buffer, map, size, valueSerializer); } else { - genericJavaRead(fury, buffer, map, size); + genericJavaRead(fury, buffer, map, size); } } diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapChunkWriter.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapChunkWriter.java index 5b69638be6..bf0178ad3a 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapChunkWriter.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapChunkWriter.java @@ -42,24 +42,26 @@ public MapChunkWriter(Fury fury) { * preserve two byte for header and chunk size and record the write index * so that we can write key value at first, write header and chunk size when the chunk is finish at correct position */ - private boolean preserveByteForHeaderAndChunkSize = false; + private boolean hasPreservedByte = false; private void preserveByteForHeaderAndChunkSize(MemoryBuffer memoryBuffer) { - if (!preserveByteForHeaderAndChunkSize) { + if (hasPreservedByte) { return; } int writerIndex = memoryBuffer.writerIndex(); // preserve two byte for header and chunk size memoryBuffer.writerIndex(writerIndex + 2); this.startOffset = writerIndex; - preserveByteForHeaderAndChunkSize = true; + hasPreservedByte = true; } public MapChunkWriter next(Object key, Object value, MemoryBuffer buffer) { if (!markChunkWriteFinish) { - if (key == null && chunkSize > 0) { + if (key == null) { prevKeyIsNull = true; - reset(buffer); + if (chunkSize > 0) { + reset(buffer); + } } if (prevKeyIsNull && key != null) { reset(buffer); @@ -100,7 +102,6 @@ public void increaseChunkSize() { } - public void generalChunkWrite(Object key, Object value, MemoryBuffer memoryBuffer, ClassResolver classResolver, RefResolver refResolver, ClassInfoHolder keyClassInfoWriteCache, ClassInfoHolder valueClassInfoWriteCache) { final boolean trackingRef = fury.trackingRef(); writeKey(key, memoryBuffer, classResolver, refResolver, trackingRef, keyClassInfoWriteCache); @@ -209,33 +210,30 @@ private boolean judgeValueIsNotSameType(Object value) { public void writeKey(Object key, MemoryBuffer buffer, ClassResolver classResolver, RefResolver refResolver, boolean trackingKeyRef, ClassInfoHolder keyClassInfoWriteCache) { preserveByteForHeaderAndChunkSize(buffer); + updateKeyHeader(key, trackingKeyRef, keyIsNotSameType); //todo hening key == null提到外面？ if (!trackingKeyRef) { if (key == null) { - updateKeyHeader(null, buffer, keyClassInfoWriteCache, false, false); buffer.writeByte(Fury.NULL_FLAG); } else { if (!keyIsNotSameType) { - updateKeyHeader(key, buffer, keyClassInfoWriteCache, false, false); + writeKeyClass(key, buffer, keyClassInfoWriteCache); keyClassInfoWriteCache.getSerializer().write(buffer, key); } else { - updateKeyHeader(key, buffer, keyClassInfoWriteCache, false, true); fury.writeNonRef(buffer, key, classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache)); } } } else { //todo 提到外面 if (key == null) { - updateKeyHeader(null, buffer, keyClassInfoWriteCache, true, false); buffer.writeByte(Fury.NULL_FLAG); } else { if (!keyIsNotSameType) { - updateKeyHeader(key, buffer, keyClassInfoWriteCache, true, false); //todo key is not null, no need to write no null flag + writeKeyClass(key, buffer, keyClassInfoWriteCache); fury.writeRef(buffer, key, keyClassInfoWriteCache.getSerializer()); } else { // todo hening remove write class - updateKeyHeader(key, buffer, keyClassInfoWriteCache, true, true); if (!refResolver.writeNullFlag(buffer, key)) { fury.writeRef(buffer, key, classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache)); } @@ -250,12 +248,12 @@ public Object readKey(int header, MemoryBuffer memoryBuffer, ClassResolver class if (!trackingKeyRef) { if (keyHasNull()) { byte nullFlag = memoryBuffer.readByte(); - Preconditions.checkArgument(nullFlag != Fury.NULL_FLAG, "unexpected error"); + Preconditions.checkArgument(nullFlag == Fury.NULL_FLAG, "unexpected error"); return null; } else { if (!keyIsNotSameType()) { if (keyClassInfoReadCache.getSerializer() == null) { - classResolver.readClassInfo(memoryBuffer, keyClassInfoReadCache); + keyClassInfoReadCache.classInfo = classResolver.readClassInfo(memoryBuffer, keyClassInfoReadCache); } return keyClassInfoReadCache.getSerializer().read(memoryBuffer); } else { @@ -265,12 +263,12 @@ public Object readKey(int header, MemoryBuffer memoryBuffer, ClassResolver class } else { if (keyHasNull()) { byte nullFlag = memoryBuffer.readByte(); - Preconditions.checkArgument(nullFlag != Fury.NULL_FLAG, "unexpected error"); + Preconditions.checkArgument(nullFlag == Fury.NULL_FLAG, "unexpected error"); return null; } else { if (!keyIsNotSameType()) { if (keyClassInfoReadCache.getSerializer() == null) { - classResolver.readClassInfo(memoryBuffer, keyClassInfoReadCache); + keyClassInfoReadCache.classInfo = classResolver.readClassInfo(memoryBuffer, keyClassInfoReadCache); } return fury.readRef(memoryBuffer, keyClassInfoReadCache.getSerializer()); } else { @@ -285,28 +283,29 @@ public void writeValue(Object value, MemoryBuffer buffer, ClassResolver classRes preserveByteForHeaderAndChunkSize(buffer); if (!trackingValueRef) { if (value == null) { - updateValueHeader(null, buffer, valueClassInfoWriteCache, false, false); + updateValueHeader(null, false, false); buffer.writeByte(Fury.NULL_FLAG); } else { - updateValueHeader(value, buffer, valueClassInfoWriteCache, false, valueIsNotSameType); + updateValueHeader(value, false, valueIsNotSameType); if (!valueIsNotSameType) { - writeValueClass(value, buffer, valueClassInfoWriteCache); if (!valueHasNull()) { + writeValueClass(value, buffer, valueClassInfoWriteCache); valueClassInfoWriteCache.getSerializer().write(buffer, value); } else { buffer.writeByte(Fury.NOT_NULL_VALUE_FLAG); + writeValueClass(value, buffer, valueClassInfoWriteCache); valueClassInfoWriteCache.getSerializer().write(buffer, value); } } else { - fury.writeNonRef(buffer, value, classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache)); + fury.writeNullable(buffer, value, classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache)); } } } else { if (value == null) { - updateValueHeader(null, buffer, valueClassInfoWriteCache, true, false); + updateValueHeader(null, true, false); buffer.writeByte(Fury.NULL_FLAG); } else { - updateKeyHeader(value, buffer, valueClassInfoWriteCache, true, valueIsNotSameType); + updateKeyHeader(value, true, valueIsNotSameType); if (!valueIsNotSameType) { writeValueClass(value, buffer, valueClassInfoWriteCache); if (!valueHasNull()) { @@ -328,35 +327,43 @@ public void writeValue(Object value, MemoryBuffer buffer, ClassResolver classRes public Object readValue(int header, MemoryBuffer buffer, ClassResolver classResolver, boolean trackingValueRef, ClassInfoHolder valueClassInfoReadCache) { this.header = header; if (!trackingValueRef) { - if (valueHasNull()) { - byte flag = buffer.readByte(); - if (flag == Fury.NOT_NULL_VALUE_FLAG) { + if (!valueIsNotSameType()) { + if (valueHasNull()) { + byte flag = buffer.readByte(); + if (flag == Fury.NOT_NULL_VALUE_FLAG) { + if (valueClassInfoReadCache.getSerializer() == null) { + valueClassInfoReadCache.classInfo = classResolver.readClassInfo(buffer, valueClassInfoReadCache); + } + return valueClassInfoReadCache.getSerializer().read(buffer); + } else { + return null; + } + } else { if (valueClassInfoReadCache.getSerializer() == null) { - classResolver.readClassInfo(buffer, valueClassInfoReadCache); + valueClassInfoReadCache.classInfo = classResolver.readClassInfo(buffer, valueClassInfoReadCache); } return valueClassInfoReadCache.getSerializer().read(buffer); - } else { - return null; } } else { - return valueClassInfoReadCache.getSerializer().read(buffer); + return fury.readNullable(buffer, valueClassInfoReadCache); } + } else { - if (valueClassInfoReadCache.getSerializer() == null) { - classResolver.readClassInfo(buffer, valueClassInfoReadCache); - } if (!valueIsNotSameType) { if (valueHasNull()) { byte flag = buffer.readByte(); if (flag == Fury.NOT_NULL_VALUE_FLAG) { if (valueClassInfoReadCache.getSerializer() == null) { - classResolver.readClassInfo(buffer, valueClassInfoReadCache); + valueClassInfoReadCache.classInfo = classResolver.readClassInfo(buffer, valueClassInfoReadCache); } return fury.readRef(buffer, valueClassInfoReadCache.getSerializer()); } else { return null; } } else { + if (valueClassInfoReadCache.getSerializer() == null) { + valueClassInfoReadCache.classInfo = classResolver.readClassInfo(buffer, valueClassInfoReadCache); + } return fury.readRef(buffer, valueClassInfoReadCache.getSerializer()); } } else { @@ -464,7 +471,7 @@ public Object readValue(int header, MemoryBuffer buffer, ClassResolver classReso // // } - private void updateKeyHeader(Object key, MemoryBuffer memoryBuffer, ClassInfoHolder keyClassInfoWriteCache, boolean trackingKeyRef, boolean keyIsNotSameType) { + private void updateKeyHeader(Object key, boolean trackingKeyRef, boolean keyIsNotSameType) { if (key == null) { header |= MapFlags.KEY_HAS_NULL; } else { @@ -479,18 +486,18 @@ private void updateKeyHeader(Object key, MemoryBuffer memoryBuffer, ClassInfoHol } private void writeKeyClass(Object key, MemoryBuffer memoryBuffer, ClassInfoHolder keyClassInfoWriteCache) { - ClassResolver classResolver = fury.getClassResolver(); - ClassInfo classInfo = classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache); if (!writeKeyClassInfo) { + ClassResolver classResolver = fury.getClassResolver(); + ClassInfo classInfo = classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache); classResolver.writeClass(memoryBuffer, classInfo); writeKeyClassInfo = true; } } private void writeValueClass(Object value, MemoryBuffer memoryBuffer, ClassInfoHolder valueClassInfoWriteCache) { - ClassResolver classResolver = fury.getClassResolver(); - ClassInfo classInfo = classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache); if (!writeValueClassInfo) { + ClassResolver classResolver = fury.getClassResolver(); + ClassInfo classInfo = classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache); classResolver.writeClass(memoryBuffer, classInfo); writeValueClassInfo = true; } @@ -516,7 +523,7 @@ private void updateFinalValueHeader(Object value, boolean trackingValueRef) { } } - private void updateValueHeader(Object value, MemoryBuffer memoryBuffer, ClassInfoHolder valueClassInfoWriteCache, boolean trackingValueRef, boolean valueIsNotSameType) { + private void updateValueHeader(Object value, boolean trackingValueRef, boolean valueIsNotSameType) { if (value == null) { header |= MapFlags.VALUE_HAS_NULL; } else { @@ -570,7 +577,7 @@ public void reset(MemoryBuffer memoryBuffer) { writeHeader(memoryBuffer); header = 0; chunkSize = 0; - preserveByteForHeaderAndChunkSize = false; + hasPreservedByte = false; writeKeyClassInfo = false; writeValueClassInfo = false; prevKeyIsNull = false; @@ -646,7 +653,7 @@ private boolean valueHasNull() { return (header & MapFlags.VALUE_HAS_NULL) == MapFlags.VALUE_HAS_NULL; } - private boolean valueNotSameType() { + private boolean valueIsNotSameType() { return (header & MapFlags.VALUE_NOT_SAME_TYPE) == MapFlags.VALUE_NOT_SAME_TYPE; } @@ -659,5 +666,4 @@ public boolean isMarkChunkWriteFinish() { } - } From ca2d221adc59c933f479b1944422d74b77e1f24b Mon Sep 17 00:00:00 2001 From: hening Date: Tue, 30 Jul 2024 14:49:41 +0800 Subject: [PATCH 09/18] debug --- .../collection/AbstractMapSerializer.java | 44 +++++++++----- .../serializer/collection/MapChunkWriter.java | 57 +++++++++++-------- .../collection/MapSerializersTest.java | 28 +++++++++ .../org/apache/fury/test/bean/MapFields.java | 1 + 4 files changed, 91 insertions(+), 39 deletions(-) diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java index 55777b5f4f..fe36e3598f 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java @@ -167,8 +167,8 @@ private void javaWriteWithKVSerializers( Object key = entry.getKey(); Object value = entry.getValue(); mapChunkWriter = mapChunkWriter.next(entry.getKey(), entry.getValue(), buffer); - mapChunkWriter.writeFinalValue(value, buffer, valueSerializer); mapChunkWriter.writeFinalKey(key, buffer, keySerializer); + mapChunkWriter.writeFinalValue(value, buffer, valueSerializer); mapChunkWriter.increaseChunkSize(); // fury.writeRef(buffer, key, keySerializer); // fury.writeRef(buffer, value, valueSerializer); @@ -625,6 +625,8 @@ private void javaKVTypesFinalChunkRead(Fury fury, while (size > 0) { byte chunkSize = buffer.readByte(); byte header = buffer.readByte(); + mapChunkWriter.setKeySerializer(null); + mapChunkWriter.setValueSerializer(null); Preconditions.checkArgument(chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); for (byte i = 0; i < chunkSize; i++) { Object key; @@ -679,7 +681,9 @@ private void javaKeyTypeFinalChunkRead(Fury fury, Preconditions.checkArgument(chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); byte header = buffer.readByte(); valueClassInfoReadCache.classInfo = classResolver.nilClassInfo(); - while(chunkSize > 0) { + mapChunkWriter.setKeySerializer(null); + mapChunkWriter.setValueSerializer(null); + while (chunkSize > 0) { generics.pushGenericType(keyGenericType); Object key = mapChunkWriter.readFinalKey(buffer, header, keySerializer); generics.pushGenericType(valueGenericType); @@ -731,6 +735,8 @@ private void javaValueTypeFinalChunkRead(Fury fury byte chunkSize = buffer.readByte(); Preconditions.checkArgument(chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); byte header = buffer.readByte(); + mapChunkWriter.setKeySerializer(null); + mapChunkWriter.setValueSerializer(null); valueClassInfoReadCache.classInfo = classResolver.nilClassInfo(); while(chunkSize > 0) { generics.pushGenericType(keyGenericType); @@ -785,18 +791,22 @@ private void javaKVTypesNonFinalChunkRead(Fury fury, byte chunkSize = buffer.readByte(); Preconditions.checkArgument(chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); if (chunkSize == 0) { - generics.pushGenericType(keyGenericType); - Object key = readJavaRefOptimized(fury, refResolver, trackingKeyRef, buffer, keyClassInfoReadCache); - generics.popGenericType(); - generics.pushGenericType(valueGenericType); - Object value = readJavaRefOptimized(fury, refResolver, trackingValueRef, buffer, valueClassInfoReadCache); - generics.popGenericType(); - map.put(key, value); - size--; + while (size > 0) { + generics.pushGenericType(keyGenericType); + Object key = readJavaRefOptimized(fury, refResolver, trackingKeyRef, buffer, keyClassInfoReadCache); + generics.popGenericType(); + generics.pushGenericType(valueGenericType); + Object value = readJavaRefOptimized(fury, refResolver, trackingValueRef, buffer, valueClassInfoReadCache); + generics.popGenericType(); + map.put(key, value); + size--; + } } else { byte header = buffer.readByte(); keyClassInfoReadCache.classInfo = classResolver.nilClassInfo(); valueClassInfoReadCache.classInfo = classResolver.nilClassInfo(); + mapChunkWriter.setKeySerializer(null); + mapChunkWriter.setValueSerializer(null); while(chunkSize > 0) { generics.pushGenericType(keyGenericType); Object key = mapChunkWriter.readKey(header, buffer, classResolver, trackingKeyRef, keyClassInfoReadCache); @@ -847,15 +857,19 @@ private void javaChunkRead(Fury fury, MemoryBuffer buffer, Map map, int size) { byte chunkSize = buffer.readByte(); Preconditions.checkArgument(chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); if (chunkSize == 0) { - Object key = fury.readRef(buffer, keyClassInfoReadCache); - Object value = fury.readRef(buffer, keyClassInfoReadCache); - map.put(key, value); - size--; + while (size > 0) { + Object key = fury.readRef(buffer, keyClassInfoReadCache); + Object value = fury.readRef(buffer, keyClassInfoReadCache); + map.put(key, value); + size--; + } } else { byte header = buffer.readByte(); + mapChunkWriter.setKeySerializer(null); + mapChunkWriter.setValueSerializer(null); keyClassInfoReadCache.classInfo = classResolver.nilClassInfo(); valueClassInfoReadCache.classInfo = classResolver.nilClassInfo(); - while(chunkSize > 0) { + while (chunkSize > 0) { Object key = mapChunkWriter.readKey(header, buffer, classResolver, trackingKeyRef, keyClassInfoReadCache); Object value = mapChunkWriter.readValue(header, buffer, classResolver, trackingKeyRef, valueClassInfoReadCache); chunkSize--; diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapChunkWriter.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapChunkWriter.java index bf0178ad3a..d638c39272 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapChunkWriter.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapChunkWriter.java @@ -33,6 +33,8 @@ public MapChunkWriter(Fury fury) { private boolean keyIsNotSameType = false; private boolean valueIsNotSameType = false; private boolean prevKeyIsNull = false; + private Serializer keySerializer; + private Serializer valueSerializer; /** * mark chunk write finish @@ -252,10 +254,10 @@ public Object readKey(int header, MemoryBuffer memoryBuffer, ClassResolver class return null; } else { if (!keyIsNotSameType()) { - if (keyClassInfoReadCache.getSerializer() == null) { - keyClassInfoReadCache.classInfo = classResolver.readClassInfo(memoryBuffer, keyClassInfoReadCache); + if (keySerializer == null) { + keySerializer = classResolver.readClassInfo(memoryBuffer, keyClassInfoReadCache).getSerializer(); } - return keyClassInfoReadCache.getSerializer().read(memoryBuffer); + return keySerializer.read(memoryBuffer); } else { return fury.readNonRef(memoryBuffer, keyClassInfoReadCache); } @@ -267,10 +269,10 @@ public Object readKey(int header, MemoryBuffer memoryBuffer, ClassResolver class return null; } else { if (!keyIsNotSameType()) { - if (keyClassInfoReadCache.getSerializer() == null) { - keyClassInfoReadCache.classInfo = classResolver.readClassInfo(memoryBuffer, keyClassInfoReadCache); + if (keySerializer == null) { + keySerializer = classResolver.readClassInfo(memoryBuffer, keyClassInfoReadCache).getSerializer(); } - return fury.readRef(memoryBuffer, keyClassInfoReadCache.getSerializer()); + return fury.readRef(memoryBuffer, keySerializer); } else { return fury.readRef(memoryBuffer, keyClassInfoReadCache); } @@ -305,7 +307,7 @@ public void writeValue(Object value, MemoryBuffer buffer, ClassResolver classRes updateValueHeader(null, true, false); buffer.writeByte(Fury.NULL_FLAG); } else { - updateKeyHeader(value, true, valueIsNotSameType); + updateValueHeader(value, true, valueIsNotSameType); if (!valueIsNotSameType) { writeValueClass(value, buffer, valueClassInfoWriteCache); if (!valueHasNull()) { @@ -331,40 +333,40 @@ public Object readValue(int header, MemoryBuffer buffer, ClassResolver classReso if (valueHasNull()) { byte flag = buffer.readByte(); if (flag == Fury.NOT_NULL_VALUE_FLAG) { - if (valueClassInfoReadCache.getSerializer() == null) { - valueClassInfoReadCache.classInfo = classResolver.readClassInfo(buffer, valueClassInfoReadCache); + if (valueSerializer == null) { + valueSerializer = classResolver.readClassInfo(buffer, valueClassInfoReadCache).getSerializer(); } - return valueClassInfoReadCache.getSerializer().read(buffer); + return valueSerializer.read(buffer); } else { return null; } } else { - if (valueClassInfoReadCache.getSerializer() == null) { - valueClassInfoReadCache.classInfo = classResolver.readClassInfo(buffer, valueClassInfoReadCache); + if (valueSerializer == null) { + valueSerializer = classResolver.readClassInfo(buffer, valueClassInfoReadCache).getSerializer(); } - return valueClassInfoReadCache.getSerializer().read(buffer); + return valueSerializer.read(buffer); } } else { return fury.readNullable(buffer, valueClassInfoReadCache); } } else { - if (!valueIsNotSameType) { + if (!valueIsNotSameType()) { if (valueHasNull()) { byte flag = buffer.readByte(); if (flag == Fury.NOT_NULL_VALUE_FLAG) { - if (valueClassInfoReadCache.getSerializer() == null) { - valueClassInfoReadCache.classInfo = classResolver.readClassInfo(buffer, valueClassInfoReadCache); + if (valueSerializer == null) { + valueSerializer = classResolver.readClassInfo(buffer, valueClassInfoReadCache).getSerializer(); } - return fury.readRef(buffer, valueClassInfoReadCache.getSerializer()); + return fury.readRef(buffer, valueSerializer); } else { return null; } } else { - if (valueClassInfoReadCache.getSerializer() == null) { - valueClassInfoReadCache.classInfo = classResolver.readClassInfo(buffer, valueClassInfoReadCache); + if (valueSerializer == null) { + valueSerializer = classResolver.readClassInfo(buffer, valueClassInfoReadCache).getSerializer(); } - return fury.readRef(buffer, valueClassInfoReadCache.getSerializer()); + return fury.readRef(buffer, valueSerializer); } } else { return fury.readRef(buffer, valueClassInfoReadCache); @@ -486,18 +488,18 @@ private void updateKeyHeader(Object key, boolean trackingKeyRef, boolean keyIsNo } private void writeKeyClass(Object key, MemoryBuffer memoryBuffer, ClassInfoHolder keyClassInfoWriteCache) { + ClassResolver classResolver = fury.getClassResolver(); + ClassInfo classInfo = classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache); if (!writeKeyClassInfo) { - ClassResolver classResolver = fury.getClassResolver(); - ClassInfo classInfo = classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache); classResolver.writeClass(memoryBuffer, classInfo); writeKeyClassInfo = true; } } private void writeValueClass(Object value, MemoryBuffer memoryBuffer, ClassInfoHolder valueClassInfoWriteCache) { + ClassResolver classResolver = fury.getClassResolver(); + ClassInfo classInfo = classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache); if (!writeValueClassInfo) { - ClassResolver classResolver = fury.getClassResolver(); - ClassInfo classInfo = classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache); classResolver.writeClass(memoryBuffer, classInfo); writeValueClassInfo = true; } @@ -666,4 +668,11 @@ public boolean isMarkChunkWriteFinish() { } + public void setKeySerializer(Serializer keySerializer) { + this.keySerializer = keySerializer; + } + + public void setValueSerializer(Serializer valueSerializer) { + this.valueSerializer = valueSerializer; + } } diff --git a/java/fury-core/src/test/java/org/apache/fury/serializer/collection/MapSerializersTest.java b/java/fury-core/src/test/java/org/apache/fury/serializer/collection/MapSerializersTest.java index cd9f642934..99960ff1fd 100644 --- a/java/fury-core/src/test/java/org/apache/fury/serializer/collection/MapSerializersTest.java +++ b/java/fury-core/src/test/java/org/apache/fury/serializer/collection/MapSerializersTest.java @@ -251,6 +251,7 @@ public static MapFields createMapFieldsObject() { obj.emptyMap = Collections.emptyMap(); obj.sortedEmptyMap = Collections.emptySortedMap(); obj.singletonMap = Collections.singletonMap("k", "v"); + obj.differentKeyAndValueTypeMap = createDifferentKeyAndValueTypeMap(); return obj; } @@ -364,4 +365,31 @@ public void testStringKeyMapSerializer() { serDeCheck(fury, map); } } + + @Test(dataProvider = "javaFury") + public void testDifferentKeyAndValueType(Fury fury) { + Map map = createDifferentKeyAndValueTypeMap(); + Assert.assertEquals(serDe(fury, map), map); + } + + private static Map createDifferentKeyAndValueTypeMap() { + Map map = new HashMap<>(); + map.put(null, "1"); + map.put(2, "1"); + map.put(4, "1"); + map.put(6, "1"); + map.put(7, "1"); + map.put(10, "1"); + map.put(12, "null"); + map.put(19, "null"); + map.put(11, null); + map.put(20, null); + map.put(21, 9); + map.put(22, 99); + map.put(291, 900); + map.put("292", 900); + map.put("293", 900); + map.put("23", 900); + return map; + } } diff --git a/java/fury-test-core/src/main/java/org/apache/fury/test/bean/MapFields.java b/java/fury-test-core/src/main/java/org/apache/fury/test/bean/MapFields.java index 54f2551300..7263db9d97 100644 --- a/java/fury-test-core/src/main/java/org/apache/fury/test/bean/MapFields.java +++ b/java/fury-test-core/src/main/java/org/apache/fury/test/bean/MapFields.java @@ -53,6 +53,7 @@ public class MapFields { public Map emptyMap; public Map sortedEmptyMap; public Map singletonMap; + public Map differentKeyAndValueTypeMap; public static Object copyToCanEqual(Object o, Object newInstance) { return CollectionFields.copyToCanEqual(o, newInstance); From 82cb1fbe10d6541f42623a4706d9737777b3ea69 Mon Sep 17 00:00:00 2001 From: hening Date: Tue, 30 Jul 2024 15:22:14 +0800 Subject: [PATCH 10/18] remove useless commit --- .../apache/fury/serializer/collection/MapSerializersTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/java/fury-core/src/test/java/org/apache/fury/serializer/collection/MapSerializersTest.java b/java/fury-core/src/test/java/org/apache/fury/serializer/collection/MapSerializersTest.java index 99960ff1fd..c97f3790b2 100644 --- a/java/fury-core/src/test/java/org/apache/fury/serializer/collection/MapSerializersTest.java +++ b/java/fury-core/src/test/java/org/apache/fury/serializer/collection/MapSerializersTest.java @@ -137,7 +137,6 @@ public void testTreeMap() { .withLanguage(Language.JAVA) .withRefTracking(referenceTracking) .requireClassRegistration(false) - .withCodegen(false) .build(); TreeMap map = new TreeMap<>( @@ -152,7 +151,7 @@ public void testTreeMap() { }); map.put("str1", "1"); map.put("str2", "1"); -// assertEquals(map, serDe(fury, map)); + assertEquals(map, serDe(fury, map)); BeanForMap beanForMap = new BeanForMap(); assertEquals(beanForMap, serDe(fury, beanForMap)); } From c492e8e0322ab343793f35b848c55d663b5de851 Mon Sep 17 00:00:00 2001 From: hening Date: Tue, 30 Jul 2024 17:37:14 +0800 Subject: [PATCH 11/18] delete useless code --- .../collection/AbstractMapSerializer.java | 180 +----------------- .../serializer/collection/MapChunkWriter.java | 168 +--------------- 2 files changed, 8 insertions(+), 340 deletions(-) diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java index fe36e3598f..b8917fe880 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java @@ -170,8 +170,6 @@ private void javaWriteWithKVSerializers( mapChunkWriter.writeFinalKey(key, buffer, keySerializer); mapChunkWriter.writeFinalValue(value, buffer, valueSerializer); mapChunkWriter.increaseChunkSize(); -// fury.writeRef(buffer, key, keySerializer); -// fury.writeRef(buffer, value, valueSerializer); } mapChunkWriter.writeHeader(buffer); } @@ -223,26 +221,6 @@ private void genericJavaWrite(Fury fury, MemoryBuffer buffer, Map map) { } } - private void javaKVTypesFinalWrite( - Fury fury, - MemoryBuffer buffer, - Map map, - GenericType keyGenericType, - GenericType valueGenericType, - Generics generics) { - Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); - Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); - for (Object object : map.entrySet()) { - Map.Entry entry = (Map.Entry) object; - generics.pushGenericType(keyGenericType); - fury.writeRef(buffer, entry.getKey(), keySerializer); - generics.popGenericType(); - generics.pushGenericType(valueGenericType); - fury.writeRef(buffer, entry.getValue(), valueSerializer); - generics.popGenericType(); - } - } - /** * kv final write do not need to predict , since key and value is almost same type unless null * @@ -279,35 +257,6 @@ private void javaKVTypesFinalChunkWrite( mapChunkWriter.writeHeader(buffer); } - private void javaKeyTypeFinalWrite( - Fury fury, - MemoryBuffer buffer, - Map map, - GenericType keyGenericType, - GenericType valueGenericType, - Generics generics) { - ClassResolver classResolver = fury.getClassResolver(); - RefResolver refResolver = fury.getRefResolver(); - boolean trackingValueRef = fury.getClassResolver().needToWriteRef(valueGenericType.getCls()); - Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); - for (Object object : map.entrySet()) { - Map.Entry entry = (Map.Entry) object; - generics.pushGenericType(keyGenericType); - fury.writeRef(buffer, entry.getKey(), keySerializer); - generics.popGenericType(); - generics.pushGenericType(valueGenericType); - writeJavaRefOptimized( - fury, - classResolver, - refResolver, - trackingValueRef, - buffer, - entry.getValue(), - valueClassInfoWriteCache); - generics.popGenericType(); - } - } - private void javaKeyTypeFinalChunkWrite( Fury fury, MemoryBuffer buffer, @@ -397,18 +346,6 @@ private void javaKVTypesNonFinalChunkWrite( mapChunkWriter.writeHeader(buffer); } - private void generalJavaWrite(Fury fury, MemoryBuffer buffer, Map map) { - ClassResolver classResolver = fury.getClassResolver(); - RefResolver refResolver = fury.getRefResolver(); - for (Object object : map.entrySet()) { - Map.Entry entry = (Map.Entry) object; - writeJavaRefOptimized( - fury, classResolver, refResolver, buffer, entry.getKey(), keyClassInfoWriteCache); - writeJavaRefOptimized( - fury, classResolver, refResolver, buffer, entry.getValue(), valueClassInfoWriteCache); - } - } - protected void generalJavaChunkWrite(Fury fury, MemoryBuffer buffer, Map map) { MapChunkWriter chunkWriter = new MapChunkWriter(fury); ClassResolver classResolver = fury.getClassResolver(); @@ -583,14 +520,14 @@ private void genericJavaRead(Fury fury, MemoryBuffer buffer, Map map, int size) Generics generics = fury.getGenerics(); GenericType genericType = generics.nextGenericType(); if (genericType == null) { - javaChunkRead(fury, buffer, map, size); + generalJavaChunkRead(fury, buffer, map, size); } else { GenericType keyGenericType = genericType.getTypeParameter0(); GenericType valueGenericType = genericType.getTypeParameter1(); if (genericType.getTypeParametersCount() < 2) { Tuple2 kvGenericType = getKVGenericType(genericType); if (keyGenericType == objType && valueGenericType == objType) { - javaChunkRead(fury, buffer, map, size); + generalJavaChunkRead(fury, buffer, map, size); return; } keyGenericType = kvGenericType.f0; @@ -644,27 +581,6 @@ private void javaKVTypesFinalChunkRead(Fury fury, } - private void javaKVTypesFinalRead( - Fury fury, - MemoryBuffer buffer, - Map map, - GenericType keyGenericType, - GenericType valueGenericType, - Generics generics, - int size) { - Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); - Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); - for (int i = 0; i < size; i++) { - generics.pushGenericType(keyGenericType); - Object key = fury.readRef(buffer, keySerializer); - generics.popGenericType(); - generics.pushGenericType(valueGenericType); - Object value = fury.readRef(buffer, valueSerializer); - generics.popGenericType(); - map.put(key, value); - } - } - private void javaKeyTypeFinalChunkRead(Fury fury, MemoryBuffer buffer, Map map, @@ -672,7 +588,7 @@ private void javaKeyTypeFinalChunkRead(Fury fury, GenericType valueGenericType, Generics generics, int size) { - final MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); + MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); ClassResolver classResolver = fury.getClassResolver(); boolean trackingValueRef = classResolver.needToWriteRef(valueGenericType.getCls()); Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); @@ -680,7 +596,6 @@ private void javaKeyTypeFinalChunkRead(Fury fury, byte chunkSize = buffer.readByte(); Preconditions.checkArgument(chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); byte header = buffer.readByte(); - valueClassInfoReadCache.classInfo = classResolver.nilClassInfo(); mapChunkWriter.setKeySerializer(null); mapChunkWriter.setValueSerializer(null); while (chunkSize > 0) { @@ -697,28 +612,6 @@ private void javaKeyTypeFinalChunkRead(Fury fury, } } - private void javaKeyTypeFinalRead(Fury fury - , MemoryBuffer buffer - , Map map - , GenericType keyGenericType - , GenericType valueGenericType - , Generics generics - , int size) { - RefResolver refResolver = fury.getRefResolver(); - boolean trackingValueRef = fury.getClassResolver().needToWriteRef(valueGenericType.getCls()); - Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); - for (int i = 0; i < size; i++) { - generics.pushGenericType(keyGenericType); - Object key = fury.readRef(buffer, keySerializer); - generics.popGenericType(); - generics.pushGenericType(valueGenericType); - Object value = - readJavaRefOptimized( - fury, refResolver, trackingValueRef, buffer, valueClassInfoWriteCache); - generics.popGenericType(); - map.put(key, value); - } - } private void javaValueTypeFinalChunkRead(Fury fury , MemoryBuffer buffer @@ -737,8 +630,7 @@ private void javaValueTypeFinalChunkRead(Fury fury byte header = buffer.readByte(); mapChunkWriter.setKeySerializer(null); mapChunkWriter.setValueSerializer(null); - valueClassInfoReadCache.classInfo = classResolver.nilClassInfo(); - while(chunkSize > 0) { + while (chunkSize > 0) { generics.pushGenericType(keyGenericType); Object key = mapChunkWriter.readKey(header, buffer, classResolver, trackingKeyRef, keyClassInfoReadCache); generics.pushGenericType(valueGenericType); @@ -752,28 +644,6 @@ private void javaValueTypeFinalChunkRead(Fury fury } } - private void javaValueTypeFinalRead( - Fury fury, - MemoryBuffer buffer, - Map map, - GenericType keyGenericType, - GenericType valueGenericType, - Generics generics, - int size) { - boolean trackingKeyRef = fury.getClassResolver().needToWriteRef(keyGenericType.getCls()); - Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); - RefResolver refResolver = fury.getRefResolver(); - for (int i = 0; i < size; i++) { - generics.pushGenericType(keyGenericType); - Object key = - readJavaRefOptimized(fury, refResolver, trackingKeyRef, buffer, keyClassInfoWriteCache); - generics.popGenericType(); - generics.pushGenericType(valueGenericType); - Object value = fury.readRef(buffer, valueSerializer); - generics.popGenericType(); - map.put(key, value); - } - } private void javaKVTypesNonFinalChunkRead(Fury fury, MemoryBuffer buffer, @@ -803,11 +673,9 @@ private void javaKVTypesNonFinalChunkRead(Fury fury, } } else { byte header = buffer.readByte(); - keyClassInfoReadCache.classInfo = classResolver.nilClassInfo(); - valueClassInfoReadCache.classInfo = classResolver.nilClassInfo(); mapChunkWriter.setKeySerializer(null); mapChunkWriter.setValueSerializer(null); - while(chunkSize > 0) { + while (chunkSize > 0) { generics.pushGenericType(keyGenericType); Object key = mapChunkWriter.readKey(header, buffer, classResolver, trackingKeyRef, keyClassInfoReadCache); generics.popGenericType(); @@ -822,34 +690,8 @@ private void javaKVTypesNonFinalChunkRead(Fury fury, } } - private void javaKVTypesNonFinalRead( - Fury fury, - MemoryBuffer buffer, - Map map, - GenericType keyGenericType, - GenericType valueGenericType, - Generics generics, - int size) { - ClassResolver classResolver = fury.getClassResolver(); - RefResolver refResolver = fury.getRefResolver(); - boolean trackingKeyRef = classResolver.needToWriteRef(keyGenericType.getCls()); - boolean trackingValueRef = classResolver.needToWriteRef(valueGenericType.getCls()); - for (int i = 0; i < size; i++) { - generics.pushGenericType(keyGenericType); - Object key = - readJavaRefOptimized(fury, refResolver, trackingKeyRef, buffer, keyClassInfoWriteCache); - generics.popGenericType(); - generics.pushGenericType(valueGenericType); - Object value = - readJavaRefOptimized( - fury, refResolver, trackingValueRef, buffer, valueClassInfoWriteCache); - generics.popGenericType(); - map.put(key, value); - } - } - - private void javaChunkRead(Fury fury, MemoryBuffer buffer, Map map, int size) { + private void generalJavaChunkRead(Fury fury, MemoryBuffer buffer, Map map, int size) { final MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); ClassResolver classResolver = fury.getClassResolver(); boolean trackingKeyRef = fury.trackingRef(); @@ -867,8 +709,6 @@ private void javaChunkRead(Fury fury, MemoryBuffer buffer, Map map, int size) { byte header = buffer.readByte(); mapChunkWriter.setKeySerializer(null); mapChunkWriter.setValueSerializer(null); - keyClassInfoReadCache.classInfo = classResolver.nilClassInfo(); - valueClassInfoReadCache.classInfo = classResolver.nilClassInfo(); while (chunkSize > 0) { Object key = mapChunkWriter.readKey(header, buffer, classResolver, trackingKeyRef, keyClassInfoReadCache); Object value = mapChunkWriter.readValue(header, buffer, classResolver, trackingKeyRef, valueClassInfoReadCache); @@ -880,14 +720,6 @@ private void javaChunkRead(Fury fury, MemoryBuffer buffer, Map map, int size) { } } - private void generalJavaRead(Fury fury, MemoryBuffer buffer, Map map, int size) { - for (int i = 0; i < size; i++) { - Object key = fury.readRef(buffer, keyClassInfoReadCache); - Object value = fury.readRef(buffer, valueClassInfoReadCache); - map.put(key, value); - } - } - @SuppressWarnings("unchecked") public static void xreadElements(Fury fury, MemoryBuffer buffer, Map map, int size) { Generics generics = fury.getGenerics(); diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapChunkWriter.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapChunkWriter.java index d638c39272..7fa41d56c0 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapChunkWriter.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapChunkWriter.java @@ -11,9 +11,6 @@ import java.util.Map; -/** - * todo 1 value如果有空的情况，value可能有多个空，写class信息，在哪里写，2 写class当前可能写重复了 - */ public class MapChunkWriter { private static final int MAX_CHUNK_SIZE = 127; @@ -103,7 +100,6 @@ public void increaseChunkSize() { chunkSize++; } - public void generalChunkWrite(Object key, Object value, MemoryBuffer memoryBuffer, ClassResolver classResolver, RefResolver refResolver, ClassInfoHolder keyClassInfoWriteCache, ClassInfoHolder valueClassInfoWriteCache) { final boolean trackingRef = fury.trackingRef(); writeKey(key, memoryBuffer, classResolver, refResolver, trackingRef, keyClassInfoWriteCache); @@ -235,7 +231,6 @@ public void writeKey(Object key, MemoryBuffer buffer, ClassResolver classResolve writeKeyClass(key, buffer, keyClassInfoWriteCache); fury.writeRef(buffer, key, keyClassInfoWriteCache.getSerializer()); } else { - // todo hening remove write class if (!refResolver.writeNullFlag(buffer, key)) { fury.writeRef(buffer, key, classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache)); } @@ -283,12 +278,11 @@ public Object readKey(int header, MemoryBuffer memoryBuffer, ClassResolver class public void writeValue(Object value, MemoryBuffer buffer, ClassResolver classResolver, RefResolver refResolver, boolean trackingValueRef, ClassInfoHolder valueClassInfoWriteCache) { preserveByteForHeaderAndChunkSize(buffer); + updateValueHeader(value, trackingValueRef, valueIsNotSameType); if (!trackingValueRef) { if (value == null) { - updateValueHeader(null, false, false); buffer.writeByte(Fury.NULL_FLAG); } else { - updateValueHeader(value, false, valueIsNotSameType); if (!valueIsNotSameType) { if (!valueHasNull()) { writeValueClass(value, buffer, valueClassInfoWriteCache); @@ -304,10 +298,8 @@ public void writeValue(Object value, MemoryBuffer buffer, ClassResolver classRes } } else { if (value == null) { - updateValueHeader(null, true, false); buffer.writeByte(Fury.NULL_FLAG); } else { - updateValueHeader(value, true, valueIsNotSameType); if (!valueIsNotSameType) { writeValueClass(value, buffer, valueClassInfoWriteCache); if (!valueHasNull()) { @@ -375,104 +367,6 @@ public Object readValue(int header, MemoryBuffer buffer, ClassResolver classReso } -// private void writeValueNonNull(Object value, MemoryBuffer buffer, ClassResolver classResolver, RefResolver refResolver, ClassInfoHolder valueClassInfoWriteCache, boolean trackingValueRef) { -// updateValueHeader(value, buffer, valueClassInfoWriteCache, trackingValueRef); -// if (!trackingValueRef) { -// if (!valueNotSameType()) { -// Serializer valueSerializer = valueClassInfoWriteCache.getSerializer(); -// if (valueHasNull()) { -// buffer.writeByte(Fury.NOT_NULL_VALUE_FLAG); -// valueSerializer.write(buffer, value); -// } else { -// valueSerializer.write(buffer, value); -// } -// } else { -// if (keyIsNotSameType()) { -// markChunkWriteFinish(buffer); -// } -// writeJavaRefOptimized(fury, classResolver, refResolver, false, buffer, value, valueClassInfoWriteCache); -// } -// } else { -// if (!valueNotSameType()) { -// if (!refResolver.writeRefOrNull(buffer, value)) { -// valueClassInfoWriteCache.getSerializer().write(buffer, value); -// } -// } else { -// if (keyIsNotSameType()) { -// markChunkWriteFinish(buffer); -// } -// writeJavaRefOptimized(fury, classResolver, refResolver, true, buffer, value, valueClassInfoWriteCache); -// } -// } -// } - - -// public void writeKey(Object key, MemoryBuffer buffer, ClassResolver classResolver, RefResolver refResolver, ClassInfoHolder keyClassInfoWriteCache) { -// preserveByteForHeaderAndChunkSize(buffer); -// final boolean trackingRef = fury.trackingRef(); -// if (!trackingRef) { -// if (key == null) { -// header |= MapFlags.KEY_HAS_NULL; -// buffer.writeByte(Fury.NULL_FLAG); -// } else { -// updateKeyHeader(key, buffer, keyClassInfoWriteCache, false, keyIsNotSameType); -// if (!keyIsNotSameType) { -// keyClassInfoWriteCache.getSerializer().write(buffer, key); -// } else { -// fury.writeNonRef(buffer, key, classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache)); -// } -// } -// } else { -// if (key == null) { -// //todo remove writeClass -// updateKeyHeader(null, buffer, keyClassInfoWriteCache, true, false); -// buffer.writeByte(Fury.NULL_FLAG); -// } else { -// if (!keyIsNotSameType) { -// ClassInfo classInfo = classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache); -// boolean trackingKeyRef = classInfo.getSerializer().needToWriteRef(); -// updateKeyHeader(key, buffer, keyClassInfoWriteCache, trackingKeyRef, false); -// fury.writeRef(buffer, key, keyClassInfoWriteCache.getSerializer()); -// } else { -// writeJavaRefOptimized(fury, classResolver, refResolver, buffer, key, keyClassInfoWriteCache); -// } -// } -// } -// } -// -// -// public void writeValue(Object value -// , MemoryBuffer buffer -// , ClassResolver classResolver -// , RefResolver refResolver -// , ClassInfoHolder valueClassInfoWriteCache) { -// preserveByteForHeaderAndChunkSize(buffer); -// boolean trackingRef = fury.trackingRef(); -// if (!trackingRef) { -// if (value == null) { -// header |= MapFlags.VALUE_HAS_NULL; -// buffer.writeByte(Fury.NULL_FLAG); -// } else { -// updateValueHeader(value, buffer, valueClassInfoWriteCache, false, valueIsNotSameType); -// if (!valueIsNotSameType) { -// if (!valueHasNull()) { -// valueClassInfoWriteCache.getSerializer().write(buffer, value); -// } else { -// buffer.writeByte(Fury.NOT_NULL_VALUE_FLAG); -// valueClassInfoWriteCache.getSerializer().write(buffer, value); -// } -// } else { -// fury.writeNonRef(buffer, value, classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache)); -// } -// } -// } else { -// ClassInfo classInfo = classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache); -// boolean trackingValueRef = classInfo.getSerializer().needToWriteRef(); -// writeValueNonNull(value, buffer, classResolver, refResolver, valueClassInfoWriteCache, trackingValueRef); -// } -// -// } - private void updateKeyHeader(Object key, boolean trackingKeyRef, boolean keyIsNotSameType) { if (key == null) { header |= MapFlags.KEY_HAS_NULL; @@ -587,65 +481,6 @@ public void reset(MemoryBuffer memoryBuffer) { valueClass = null; } - private void writeJavaRefOptimized( - Fury fury, - ClassResolver classResolver, - RefResolver refResolver, - boolean trackingRef, - MemoryBuffer buffer, - Object obj, - ClassInfoHolder classInfoHolder) { - if (trackingRef) { - if (!refResolver.writeNullFlag(buffer, obj)) { - fury.writeRef(buffer, obj, classResolver.getClassInfo(obj.getClass(), classInfoHolder)); - } - } else { - if (obj == null) { - buffer.writeByte(Fury.NULL_FLAG); - } else { - buffer.writeByte(Fury.NOT_NULL_VALUE_FLAG); - fury.writeNonRef(buffer, obj, classResolver.getClassInfo(obj.getClass(), classInfoHolder)); - } - } - } - - private void writeJavaRefOptimized( - Fury fury, - ClassResolver classResolver, - RefResolver refResolver, - MemoryBuffer buffer, - Object obj, - ClassInfoHolder classInfoHolder) { - if (!refResolver.writeNullFlag(buffer, obj)) { - fury.writeRef(buffer, obj, classResolver.getClassInfo(obj.getClass(), classInfoHolder)); - } - } - - private Object readJavaRefOptimized( - Fury fury, - RefResolver refResolver, - boolean trackingRef, - MemoryBuffer buffer, - ClassInfoHolder classInfoHolder) { - if (trackingRef) { - int nextReadRefId = refResolver.tryPreserveRefId(buffer); - if (nextReadRefId >= Fury.NOT_NULL_VALUE_FLAG) { - Object obj = fury.readNonRef(buffer, classInfoHolder); - refResolver.setReadObject(nextReadRefId, obj); - return obj; - } else { - return refResolver.getReadObject(); - } - } else { - byte headFlag = buffer.readByte(); - if (headFlag == Fury.NULL_FLAG) { - return null; - } else { - return fury.readNonRef(buffer, classInfoHolder); - } - } - } - private boolean keyHasNull() { return (header & MapFlags.KEY_HAS_NULL) == MapFlags.KEY_HAS_NULL; @@ -675,4 +510,5 @@ public void setKeySerializer(Serializer keySerializer) { public void setValueSerializer(Serializer valueSerializer) { this.valueSerializer = valueSerializer; } + } From 77c636414ab75805f1a47c442e7b3f035d61e208 Mon Sep 17 00:00:00 2001 From: hening Date: Wed, 31 Jul 2024 11:48:47 +0800 Subject: [PATCH 12/18] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E9=85=8D=E7=BD=AEisChu?= =?UTF-8?q?nkSerializeMapEnabled=EF=BC=8C=E6=98=AF=E5=90=A6=E5=BC=80?= =?UTF-8?q?=E5=90=AFmapchunk=E5=BA=8F=E5=88=97=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/java/org/apache/fury/Fury.java | 4 + .../java/org/apache/fury/config/Config.java | 7 + .../org/apache/fury/config/FuryBuilder.java | 11 + .../collection/AbstractMapSerializer.java | 1958 ++++++++++------- .../serializer/collection/MapSerializer.java | 6 +- 5 files changed, 1196 insertions(+), 790 deletions(-) diff --git a/java/fury-core/src/main/java/org/apache/fury/Fury.java b/java/fury-core/src/main/java/org/apache/fury/Fury.java index 62fb87a0f8..9e3aa97a53 100644 --- a/java/fury-core/src/main/java/org/apache/fury/Fury.java +++ b/java/fury-core/src/main/java/org/apache/fury/Fury.java @@ -1389,6 +1389,10 @@ public Class getDefaultJDKStreamSerializerType() { return config.getDefaultJDKStreamSerializerType(); } + public boolean isChunkSerializeMapEnabled() { + return config.isChunkSerializeMapEnabled(); + } + public boolean compressString() { return config.compressString(); } diff --git a/java/fury-core/src/main/java/org/apache/fury/config/Config.java b/java/fury-core/src/main/java/org/apache/fury/config/Config.java index 4e64e31f9e..2b6d261442 100644 --- a/java/fury-core/src/main/java/org/apache/fury/config/Config.java +++ b/java/fury-core/src/main/java/org/apache/fury/config/Config.java @@ -56,9 +56,11 @@ public class Config implements Serializable { private final boolean asyncCompilationEnabled; private final boolean deserializeNonexistentClass; private final boolean scalaOptimizationEnabled; + private final boolean chunkSerializeMapEnabled; private transient int configHash; private final boolean deserializeNonexistentEnumValueAsNull; + public Config(FuryBuilder builder) { language = builder.language; trackingRef = builder.trackingRef; @@ -89,6 +91,7 @@ public Config(FuryBuilder builder) { asyncCompilationEnabled = builder.asyncCompilationEnabled; scalaOptimizationEnabled = builder.scalaOptimizationEnabled; deserializeNonexistentEnumValueAsNull = builder.deserializeNonexistentEnumValueAsNull; + chunkSerializeMapEnabled = builder.chunkSerializeMapEnabled; } public Language getLanguage() { @@ -229,6 +232,10 @@ public boolean isAsyncCompilationEnabled() { return asyncCompilationEnabled; } + public boolean isChunkSerializeMapEnabled() { + return chunkSerializeMapEnabled; + } + /** Whether enable scala-specific serialization optimization. */ public boolean isScalaOptimizationEnabled() { return scalaOptimizationEnabled; diff --git a/java/fury-core/src/main/java/org/apache/fury/config/FuryBuilder.java b/java/fury-core/src/main/java/org/apache/fury/config/FuryBuilder.java index b7c93d8761..44a4b77b29 100644 --- a/java/fury-core/src/main/java/org/apache/fury/config/FuryBuilder.java +++ b/java/fury-core/src/main/java/org/apache/fury/config/FuryBuilder.java @@ -79,6 +79,7 @@ public final class FuryBuilder { boolean scalaOptimizationEnabled = false; boolean suppressClassRegistrationWarnings = true; boolean deserializeNonexistentEnumValueAsNull = false; + boolean chunkSerializeMapEnabled = false; MetaCompressor metaCompressor = new DeflaterMetaCompressor(); public FuryBuilder() {} @@ -304,6 +305,16 @@ public FuryBuilder withScalaOptimizationEnabled(boolean enableScalaOptimization) return this; } + /** + * use chunk by chunk method to serialize map, TODO: generate code for chunk by chunk method + * @param chunkSerializeMapEnabled + * @return + */ + public FuryBuilder withChunkSerializeMapEnable(boolean chunkSerializeMapEnabled) { + this.chunkSerializeMapEnabled = chunkSerializeMapEnabled; + return this; + } + private void finish() { if (classLoader == null) { classLoader = Thread.currentThread().getContextClassLoader(); diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java index b8917fe880..9f422257b7 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java @@ -23,6 +23,7 @@ import java.lang.invoke.MethodHandle; import java.util.Map; + import org.apache.fury.Fury; import org.apache.fury.collection.IdentityMap; import org.apache.fury.collection.Tuple2; @@ -38,188 +39,296 @@ import org.apache.fury.type.TypeUtils; import org.apache.fury.util.Preconditions; -/** Serializer for all map-like objects. */ +/** + * Serializer for all map-like objects. + */ @SuppressWarnings({"unchecked", "rawtypes"}) public abstract class AbstractMapSerializer extends Serializer { - protected MethodHandle constructor; - protected final boolean supportCodegenHook; - private Serializer keySerializer; - private Serializer valueSerializer; - protected final ClassInfoHolder keyClassInfoWriteCache; - protected final ClassInfoHolder keyClassInfoReadCache; - protected final ClassInfoHolder valueClassInfoWriteCache; - protected final ClassInfoHolder valueClassInfoReadCache; - // support map subclass whose key or value generics only are available, - // or one of types is already instantiated in subclass, ex: `Subclass implements Map` - private final IdentityMap> partialGenericKVTypeMap; - private final GenericType objType = fury.getClassResolver().buildGenericType(Object.class); - // For subclass whose kv type are instantiated already, such as - // `Subclass implements Map`. If declared `Map` doesn't specify - // instantiated kv type, then the serialization will need to write those kv - // types. Although we can extract this generics when creating the serializer, - // we can't do it when jit `Serializer` for some class which contains one of such map - // field. So we will write those extra kv classes to keep protocol consistency between - // interpreter and jit mode although it seems unnecessary. - // With kv header in future, we can write this kv classes only once, the cost won't be too much. - private int numElements; - - public AbstractMapSerializer(Fury fury, Class cls) { - this(fury, cls, !ReflectionUtils.isDynamicGeneratedCLass(cls)); - } - - public AbstractMapSerializer(Fury fury, Class cls, boolean supportCodegenHook) { - super(fury, cls); - this.supportCodegenHook = supportCodegenHook; - keyClassInfoWriteCache = fury.getClassResolver().nilClassInfoHolder(); - keyClassInfoReadCache = fury.getClassResolver().nilClassInfoHolder(); - valueClassInfoWriteCache = fury.getClassResolver().nilClassInfoHolder(); - valueClassInfoReadCache = fury.getClassResolver().nilClassInfoHolder(); - partialGenericKVTypeMap = new IdentityMap<>(); - } - - /** - * Set key serializer for next serialization, the serializer will be cleared when - * next serialization finished. - */ - public void setKeySerializer(Serializer keySerializer) { - this.keySerializer = keySerializer; - } - - /** - * Set value serializer for next serialization, the serializer will be cleared when - * next serialization finished. - */ - public void setValueSerializer(Serializer valueSerializer) { - this.valueSerializer = valueSerializer; - } - - @Override - public void write(MemoryBuffer buffer, T value) { - Map map = onMapWrite(buffer, value); - writeElements(fury, buffer, map); - } - - @Override - public void xwrite(MemoryBuffer buffer, T value) { - Map map = onMapWrite(buffer, value); - xwriteElements(fury, buffer, map); - } - - protected final void writeElements(Fury fury, MemoryBuffer buffer, Map map) { - Serializer keySerializer = this.keySerializer; - Serializer valueSerializer = this.valueSerializer; - // clear the elemSerializer to avoid conflict if the nested - // serialization has collection field. - // TODO use generics for compatible serializer. - this.keySerializer = null; - this.valueSerializer = null; - if (keySerializer != null && valueSerializer != null) { - javaWriteWithKVSerializers(fury, buffer, map, keySerializer, valueSerializer); - } else if (keySerializer != null) { - javaWriteWithKeySerializers(map, buffer, keySerializer); - } else if (valueSerializer != null) { - javaWriteWithValueSerializers(map, buffer, valueSerializer); - } else { - genericJavaWrite(fury, buffer, map); - } - } - - private void javaWriteWithKeySerializers(Map map, MemoryBuffer buffer, Serializer keySerializer) { - MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); - ClassResolver classResolver = fury.getClassResolver(); - RefResolver refResolver = fury.getRefResolver(); - for (Object object : map.entrySet()) { - Map.Entry entry = (Map.Entry) object; - mapChunkWriter = mapChunkWriter.next(entry.getKey(), entry.getValue(), buffer); - mapChunkWriter.writeFinalKey(entry.getKey(), buffer, keySerializer); - Object value = entry.getValue(); - mapChunkWriter.writeValue(value, buffer, classResolver, refResolver, fury.trackingRef(), valueClassInfoWriteCache); - mapChunkWriter.increaseChunkSize(); - } - mapChunkWriter.writeHeader(buffer); - } - - private void javaWriteWithValueSerializers(Map map, MemoryBuffer buffer, Serializer valueSerializer) { - MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); - ClassResolver classResolver = fury.getClassResolver(); - RefResolver refResolver = fury.getRefResolver(); - for (Object object : map.entrySet()) { - Map.Entry entry = (Map.Entry) object; - Object key = entry.getKey(); - mapChunkWriter = mapChunkWriter.next(entry.getKey(), entry.getValue(), buffer); - mapChunkWriter.writeKey(key, buffer, classResolver, refResolver, fury.trackingRef(), keyClassInfoWriteCache); - mapChunkWriter.writeFinalValue(entry.getValue(), buffer, valueSerializer); - mapChunkWriter.increaseChunkSize(); - } - mapChunkWriter.writeHeader(buffer); - } - - private void javaWriteWithKVSerializers( - Fury fury, - MemoryBuffer buffer, - Map map, - Serializer keySerializer, - Serializer valueSerializer) { - MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); - for (Object object : map.entrySet()) { - Map.Entry entry = (Map.Entry) object; - Object key = entry.getKey(); - Object value = entry.getValue(); - mapChunkWriter = mapChunkWriter.next(entry.getKey(), entry.getValue(), buffer); - mapChunkWriter.writeFinalKey(key, buffer, keySerializer); - mapChunkWriter.writeFinalValue(value, buffer, valueSerializer); - mapChunkWriter.increaseChunkSize(); - } - mapChunkWriter.writeHeader(buffer); - } - - private void genericJavaWrite(Fury fury, MemoryBuffer buffer, Map map) { - Generics generics = fury.getGenerics(); - GenericType genericType = generics.nextGenericType(); - if (genericType == null) { - generalJavaChunkWrite(fury, buffer, map); - } else { - GenericType keyGenericType = genericType.getTypeParameter0(); - GenericType valueGenericType = genericType.getTypeParameter1(); - // type parameters count for `Map field` will be 0; - // type parameters count for `SubMap field` which SubMap is - // `SubMap implements Map` will be 1; - if (genericType.getTypeParametersCount() < 2) { - Tuple2 kvGenericType = getKVGenericType(genericType); - if (keyGenericType == objType && valueGenericType == objType) { - generalJavaChunkWrite(fury, buffer, map); - return; - } - keyGenericType = kvGenericType.f0; - valueGenericType = kvGenericType.f1; - } - // Can't avoid push generics repeatedly in loop by stack depth, because push two - // generic type changed generics stack top, which is depth index, update stack top - // and depth will have some cost too. - // Stack depth to avoid push generics repeatedly in loop. - // Note push two generic type changed generics stack top, which is depth index, - // stack top should be updated when using for serialization k/v. - // int depth = fury.getDepth(); - // // depth + 1 to leave a slot for value generics, otherwise value generics will - // // be overwritten by nested key generics. - // fury.setDepth(depth + 1); - // generics.pushGenericType(keyGenericType); - // fury.setDepth(depth); - // generics.pushGenericType(valueGenericType); - boolean keyGenericTypeFinal = keyGenericType.isMonomorphic(); - boolean valueGenericTypeFinal = valueGenericType.isMonomorphic(); - if (keyGenericTypeFinal && valueGenericTypeFinal) { - javaKVTypesFinalChunkWrite(fury, buffer, map, keyGenericType, valueGenericType, generics); - } else if (keyGenericTypeFinal) { - javaKeyTypeFinalChunkWrite(fury, buffer, map, keyGenericType, valueGenericType, generics); - } else if (valueGenericTypeFinal) { - javaValueTypeFinalChunkWrite(fury, buffer, map, keyGenericType, valueGenericType, generics); - } else { - javaKVTypesNonFinalChunkWrite(fury, buffer, map, keyGenericType, valueGenericType, generics); - } - } - } + protected MethodHandle constructor; + protected final boolean supportCodegenHook; + private Serializer keySerializer; + private Serializer valueSerializer; + protected final ClassInfoHolder keyClassInfoWriteCache; + protected final ClassInfoHolder keyClassInfoReadCache; + protected final ClassInfoHolder valueClassInfoWriteCache; + protected final ClassInfoHolder valueClassInfoReadCache; + // support map subclass whose key or value generics only are available, + // or one of types is already instantiated in subclass, ex: `Subclass implements Map` + private final IdentityMap> partialGenericKVTypeMap; + private final GenericType objType = fury.getClassResolver().buildGenericType(Object.class); + // For subclass whose kv type are instantiated already, such as + // `Subclass implements Map`. If declared `Map` doesn't specify + // instantiated kv type, then the serialization will need to write those kv + // types. Although we can extract this generics when creating the serializer, + // we can't do it when jit `Serializer` for some class which contains one of such map + // field. So we will write those extra kv classes to keep protocol consistency between + // interpreter and jit mode although it seems unnecessary. + // With kv header in future, we can write this kv classes only once, the cost won't be too much. + private int numElements; + + public AbstractMapSerializer(Fury fury, Class cls) { + this(fury, cls, !ReflectionUtils.isDynamicGeneratedCLass(cls)); + } + + public AbstractMapSerializer(Fury fury, Class cls, boolean supportCodegenHook) { + super(fury, cls); + this.supportCodegenHook = supportCodegenHook; + keyClassInfoWriteCache = fury.getClassResolver().nilClassInfoHolder(); + keyClassInfoReadCache = fury.getClassResolver().nilClassInfoHolder(); + valueClassInfoWriteCache = fury.getClassResolver().nilClassInfoHolder(); + valueClassInfoReadCache = fury.getClassResolver().nilClassInfoHolder(); + partialGenericKVTypeMap = new IdentityMap<>(); + } + + /** + * Set key serializer for next serialization, the serializer will be cleared when + * next serialization finished. + */ + public void setKeySerializer(Serializer keySerializer) { + this.keySerializer = keySerializer; + } + + /** + * Set value serializer for next serialization, the serializer will be cleared when + * next serialization finished. + */ + public void setValueSerializer(Serializer valueSerializer) { + this.valueSerializer = valueSerializer; + } + + @Override + public void write(MemoryBuffer buffer, T value) { + Map map = onMapWrite(buffer, value); + if (fury.isChunkSerializeMapEnabled()) { + chunkWriteElements(fury, buffer, map); + } else { + writeElements(fury, buffer, map); + } + } + + @Override + public void xwrite(MemoryBuffer buffer, T value) { + Map map = onMapWrite(buffer, value); + xwriteElements(fury, buffer, map); + } + + protected final void writeElements(Fury fury, MemoryBuffer buffer, Map map) { + Serializer keySerializer = this.keySerializer; + Serializer valueSerializer = this.valueSerializer; + // clear the elemSerializer to avoid conflict if the nested + // serialization has collection field. + // TODO use generics for compatible serializer. + this.keySerializer = null; + this.valueSerializer = null; + if (keySerializer != null && valueSerializer != null) { + javaWriteWithKVSerializers(fury, buffer, map, keySerializer, valueSerializer); + } else if (keySerializer != null) { + ClassResolver classResolver = fury.getClassResolver(); + RefResolver refResolver = fury.getRefResolver(); + for (Object object : map.entrySet()) { + Map.Entry entry = (Map.Entry) object; + fury.writeRef(buffer, entry.getKey(), keySerializer); + Object value = entry.getValue(); + writeJavaRefOptimized( + fury, classResolver, refResolver, buffer, value, valueClassInfoWriteCache); + } + } else if (valueSerializer != null) { + ClassResolver classResolver = fury.getClassResolver(); + RefResolver refResolver = fury.getRefResolver(); + for (Object object : map.entrySet()) { + Map.Entry entry = (Map.Entry) object; + Object key = entry.getKey(); + writeJavaRefOptimized( + fury, classResolver, refResolver, buffer, key, keyClassInfoWriteCache); + fury.writeRef(buffer, entry.getValue(), valueSerializer); + } + } else { + genericJavaWrite(fury, buffer, map); + } + } + + protected final void chunkWriteElements(Fury fury, MemoryBuffer buffer, Map map) { + Serializer keySerializer = this.keySerializer; + Serializer valueSerializer = this.valueSerializer; + // clear the elemSerializer to avoid conflict if the nested + // serialization has collection field. + // TODO use generics for compatible serializer. + this.keySerializer = null; + this.valueSerializer = null; + if (keySerializer != null && valueSerializer != null) { + javaWriteWithKVSerializers(fury, buffer, map, keySerializer, valueSerializer); + } else if (keySerializer != null) { + javaWriteWithKeySerializers(map, buffer, keySerializer); + } else if (valueSerializer != null) { + javaWriteWithValueSerializers(map, buffer, valueSerializer); + } else { + genericJavaChunkWrite(fury, buffer, map); + } + } + + private void javaWriteWithKeySerializers(Map map, MemoryBuffer buffer, Serializer keySerializer) { + MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); + ClassResolver classResolver = fury.getClassResolver(); + RefResolver refResolver = fury.getRefResolver(); + for (Object object : map.entrySet()) { + Map.Entry entry = (Map.Entry) object; + mapChunkWriter = mapChunkWriter.next(entry.getKey(), entry.getValue(), buffer); + mapChunkWriter.writeFinalKey(entry.getKey(), buffer, keySerializer); + Object value = entry.getValue(); + mapChunkWriter.writeValue(value, buffer, classResolver, refResolver, fury.trackingRef(), valueClassInfoWriteCache); + mapChunkWriter.increaseChunkSize(); + } + mapChunkWriter.writeHeader(buffer); + } + + private void javaWriteWithValueSerializers(Map map, MemoryBuffer buffer, Serializer valueSerializer) { + MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); + ClassResolver classResolver = fury.getClassResolver(); + RefResolver refResolver = fury.getRefResolver(); + for (Object object : map.entrySet()) { + Map.Entry entry = (Map.Entry) object; + Object key = entry.getKey(); + mapChunkWriter = mapChunkWriter.next(entry.getKey(), entry.getValue(), buffer); + mapChunkWriter.writeKey(key, buffer, classResolver, refResolver, fury.trackingRef(), keyClassInfoWriteCache); + mapChunkWriter.writeFinalValue(entry.getValue(), buffer, valueSerializer); + mapChunkWriter.increaseChunkSize(); + } + mapChunkWriter.writeHeader(buffer); + } + + private void javaWriteWithKVSerializers( + Fury fury, + MemoryBuffer buffer, + Map map, + Serializer keySerializer, + Serializer valueSerializer) { + MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); + for (Object object : map.entrySet()) { + Map.Entry entry = (Map.Entry) object; + Object key = entry.getKey(); + Object value = entry.getValue(); + mapChunkWriter = mapChunkWriter.next(entry.getKey(), entry.getValue(), buffer); + mapChunkWriter.writeFinalKey(key, buffer, keySerializer); + mapChunkWriter.writeFinalValue(value, buffer, valueSerializer); + mapChunkWriter.increaseChunkSize(); + } + mapChunkWriter.writeHeader(buffer); + } + + private void genericJavaWrite(Fury fury, MemoryBuffer buffer, Map map) { + Generics generics = fury.getGenerics(); + GenericType genericType = generics.nextGenericType(); + if (genericType == null) { + generalJavaWrite(fury, buffer, map); + } else { + GenericType keyGenericType = genericType.getTypeParameter0(); + GenericType valueGenericType = genericType.getTypeParameter1(); + // type parameters count for `Map field` will be 0; + // type parameters count for `SubMap field` which SubMap is + // `SubMap implements Map` will be 1; + if (genericType.getTypeParametersCount() < 2) { + Tuple2 kvGenericType = getKVGenericType(genericType); + if (keyGenericType == objType && valueGenericType == objType) { + generalJavaWrite(fury, buffer, map); + return; + } + keyGenericType = kvGenericType.f0; + valueGenericType = kvGenericType.f1; + } + // Can't avoid push generics repeatedly in loop by stack depth, because push two + // generic type changed generics stack top, which is depth index, update stack top + // and depth will have some cost too. + // Stack depth to avoid push generics repeatedly in loop. + // Note push two generic type changed generics stack top, which is depth index, + // stack top should be updated when using for serialization k/v. + // int depth = fury.getDepth(); + // // depth + 1 to leave a slot for value generics, otherwise value generics will + // // be overwritten by nested key generics. + // fury.setDepth(depth + 1); + // generics.pushGenericType(keyGenericType); + // fury.setDepth(depth); + // generics.pushGenericType(valueGenericType); + boolean keyGenericTypeFinal = keyGenericType.isMonomorphic(); + boolean valueGenericTypeFinal = valueGenericType.isMonomorphic(); + if (keyGenericTypeFinal && valueGenericTypeFinal) { + javaKVTypesFinalWrite(fury, buffer, map, keyGenericType, valueGenericType, generics); + } else if (keyGenericTypeFinal) { + javaKeyTypeFinalWrite(fury, buffer, map, keyGenericType, valueGenericType, generics); + } else if (valueGenericTypeFinal) { + javaValueTypeFinalWrite(fury, buffer, map, keyGenericType, valueGenericType, generics); + } else { + javaKVTypesNonFinalWrite(fury, buffer, map, keyGenericType, valueGenericType, generics); + } + } + } + + private void genericJavaChunkWrite(Fury fury, MemoryBuffer buffer, Map map) { + Generics generics = fury.getGenerics(); + GenericType genericType = generics.nextGenericType(); + if (genericType == null) { + generalJavaChunkWrite(fury, buffer, map); + } else { + GenericType keyGenericType = genericType.getTypeParameter0(); + GenericType valueGenericType = genericType.getTypeParameter1(); + // type parameters count for `Map field` will be 0; + // type parameters count for `SubMap field` which SubMap is + // `SubMap implements Map` will be 1; + if (genericType.getTypeParametersCount() < 2) { + Tuple2 kvGenericType = getKVGenericType(genericType); + if (keyGenericType == objType && valueGenericType == objType) { + generalJavaChunkWrite(fury, buffer, map); + return; + } + keyGenericType = kvGenericType.f0; + valueGenericType = kvGenericType.f1; + } + // Can't avoid push generics repeatedly in loop by stack depth, because push two + // generic type changed generics stack top, which is depth index, update stack top + // and depth will have some cost too. + // Stack depth to avoid push generics repeatedly in loop. + // Note push two generic type changed generics stack top, which is depth index, + // stack top should be updated when using for serialization k/v. + // int depth = fury.getDepth(); + // // depth + 1 to leave a slot for value generics, otherwise value generics will + // // be overwritten by nested key generics. + // fury.setDepth(depth + 1); + // generics.pushGenericType(keyGenericType); + // fury.setDepth(depth); + // generics.pushGenericType(valueGenericType); + boolean keyGenericTypeFinal = keyGenericType.isMonomorphic(); + boolean valueGenericTypeFinal = valueGenericType.isMonomorphic(); + if (keyGenericTypeFinal && valueGenericTypeFinal) { + javaKVTypesFinalChunkWrite(fury, buffer, map, keyGenericType, valueGenericType, generics); + } else if (keyGenericTypeFinal) { + javaKeyTypeFinalChunkWrite(fury, buffer, map, keyGenericType, valueGenericType, generics); + } else if (valueGenericTypeFinal) { + javaValueTypeFinalChunkWrite(fury, buffer, map, keyGenericType, valueGenericType, generics); + } else { + javaKVTypesNonFinalChunkWrite(fury, buffer, map, keyGenericType, valueGenericType, generics); + } + } + } + + private void javaKVTypesFinalWrite( + Fury fury, + MemoryBuffer buffer, + Map map, + GenericType keyGenericType, + GenericType valueGenericType, + Generics generics) { + Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); + Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); + for (Object object : map.entrySet()) { + Map.Entry entry = (Map.Entry) object; + generics.pushGenericType(keyGenericType); + fury.writeRef(buffer, entry.getKey(), keySerializer); + generics.popGenericType(); + generics.pushGenericType(valueGenericType); + fury.writeRef(buffer, entry.getValue(), valueSerializer); + generics.popGenericType(); + } + } /** * kv final write do not need to predict , since key and value is almost same type unless null @@ -231,120 +340,226 @@ private void genericJavaWrite(Fury fury, MemoryBuffer buffer, Map map) { * @param valueGenericType * @param generics */ - private void javaKVTypesFinalChunkWrite( - Fury fury, - MemoryBuffer buffer, - Map map, - GenericType keyGenericType, - GenericType valueGenericType, - Generics generics) { - MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); - Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); - Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); - for (Object object : map.entrySet()) { - Map.Entry entry = (Map.Entry) object; - Object key = entry.getKey(); - Object value = entry.getValue(); - mapChunkWriter = mapChunkWriter.next(key, value, buffer); - generics.pushGenericType(keyGenericType); - mapChunkWriter.writeFinalKey(key, buffer, keySerializer); - generics.popGenericType(); - generics.pushGenericType(valueGenericType); - mapChunkWriter.writeFinalValue(value, buffer, valueSerializer); - generics.popGenericType(); - mapChunkWriter.increaseChunkSize(); - } - mapChunkWriter.writeHeader(buffer); - } - - private void javaKeyTypeFinalChunkWrite( - Fury fury, - MemoryBuffer buffer, - Map map, - GenericType keyGenericType, - GenericType valueGenericType, - Generics generics) { - MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); - ClassResolver classResolver = fury.getClassResolver(); - RefResolver refResolver = fury.getRefResolver(); - Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); - boolean trackingValueRef = fury.getClassResolver().needToWriteRef(valueGenericType.getCls()); - for (Object object : map.entrySet()) { - Map.Entry entry = (Map.Entry) object; - Object key = entry.getKey(); - Object value = entry.getValue(); - mapChunkWriter = mapChunkWriter.next(key, value, buffer); - generics.pushGenericType(keyGenericType); - mapChunkWriter.writeFinalKey(key, buffer, keySerializer); - generics.popGenericType(); - generics.pushGenericType(valueGenericType); - mapChunkWriter.writeValue(value, buffer, classResolver, refResolver, trackingValueRef, valueClassInfoWriteCache); - generics.popGenericType(); - mapChunkWriter.increaseChunkSize(); - } - mapChunkWriter.writeHeader(buffer); - } - - private void javaValueTypeFinalChunkWrite( - Fury fury, - MemoryBuffer buffer, - Map map, - GenericType keyGenericType, - GenericType valueGenericType, - Generics generics) { - MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); - ClassResolver classResolver = fury.getClassResolver(); - RefResolver refResolver = fury.getRefResolver(); - Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); - boolean trackingKeyRef = fury.getClassResolver().needToWriteRef(keyGenericType.getCls()); - for (Object object : map.entrySet()) { - Map.Entry entry = (Map.Entry) object; - Object key = entry.getKey(); - Object value = entry.getValue(); - mapChunkWriter = mapChunkWriter.next(key, value, buffer); - generics.pushGenericType(keyGenericType); - mapChunkWriter.writeKey(key, buffer, classResolver, refResolver, trackingKeyRef, keyClassInfoWriteCache); - generics.popGenericType(); - generics.pushGenericType(valueGenericType); - mapChunkWriter.writeFinalValue(value, buffer, valueSerializer); - generics.popGenericType(); - mapChunkWriter.increaseChunkSize(); - } - mapChunkWriter.writeHeader(buffer); - } - - private void javaKVTypesNonFinalChunkWrite( - Fury fury, - MemoryBuffer buffer, - Map map, - GenericType keyGenericType, - GenericType valueGenericType, - Generics generics) { - MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); - ClassResolver classResolver = fury.getClassResolver(); - RefResolver refResolver = fury.getRefResolver(); - boolean trackingKeyRef = fury.getClassResolver().needToWriteRef(keyGenericType.getCls()); - boolean trackingValueRef = fury.getClassResolver().needToWriteRef(valueGenericType.getCls()); - for (Object object : map.entrySet()) { - Map.Entry entry = (Map.Entry) object; - Object key = entry.getKey(); - Object value = entry.getValue(); - mapChunkWriter = mapChunkWriter.next(key, value, buffer); - if (mapChunkWriter.isMarkChunkWriteFinish()) { - writeJavaRefOptimized(fury, classResolver, refResolver, trackingKeyRef, buffer, key, keyClassInfoWriteCache); - writeJavaRefOptimized(fury, classResolver, refResolver, trackingValueRef, buffer, value, keyClassInfoWriteCache); - } else { + private void javaKVTypesFinalChunkWrite( + Fury fury, + MemoryBuffer buffer, + Map map, + GenericType keyGenericType, + GenericType valueGenericType, + Generics generics) { + MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); + Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); + Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); + for (Object object : map.entrySet()) { + Map.Entry entry = (Map.Entry) object; + Object key = entry.getKey(); + Object value = entry.getValue(); + mapChunkWriter = mapChunkWriter.next(key, value, buffer); generics.pushGenericType(keyGenericType); - mapChunkWriter.writeKey(key, buffer, classResolver, refResolver, trackingKeyRef, keyClassInfoWriteCache); + mapChunkWriter.writeFinalKey(key, buffer, keySerializer); + generics.popGenericType(); + generics.pushGenericType(valueGenericType); + mapChunkWriter.writeFinalValue(value, buffer, valueSerializer); + generics.popGenericType(); + mapChunkWriter.increaseChunkSize(); + } + mapChunkWriter.writeHeader(buffer); + } + + private void javaKeyTypeFinalWrite( + Fury fury, + MemoryBuffer buffer, + Map map, + GenericType keyGenericType, + GenericType valueGenericType, + Generics generics) { + ClassResolver classResolver = fury.getClassResolver(); + RefResolver refResolver = fury.getRefResolver(); + boolean trackingValueRef = fury.getClassResolver().needToWriteRef(valueGenericType.getCls()); + Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); + for (Object object : map.entrySet()) { + Map.Entry entry = (Map.Entry) object; + generics.pushGenericType(keyGenericType); + fury.writeRef(buffer, entry.getKey(), keySerializer); + generics.popGenericType(); + generics.pushGenericType(valueGenericType); + writeJavaRefOptimized( + fury, + classResolver, + refResolver, + trackingValueRef, + buffer, + entry.getValue(), + valueClassInfoWriteCache); + generics.popGenericType(); + } + } + + private void javaKeyTypeFinalChunkWrite( + Fury fury, + MemoryBuffer buffer, + Map map, + GenericType keyGenericType, + GenericType valueGenericType, + Generics generics) { + MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); + ClassResolver classResolver = fury.getClassResolver(); + RefResolver refResolver = fury.getRefResolver(); + Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); + boolean trackingValueRef = fury.getClassResolver().needToWriteRef(valueGenericType.getCls()); + for (Object object : map.entrySet()) { + Map.Entry entry = (Map.Entry) object; + Object key = entry.getKey(); + Object value = entry.getValue(); + mapChunkWriter = mapChunkWriter.next(key, value, buffer); + generics.pushGenericType(keyGenericType); + mapChunkWriter.writeFinalKey(key, buffer, keySerializer); generics.popGenericType(); generics.pushGenericType(valueGenericType); mapChunkWriter.writeValue(value, buffer, classResolver, refResolver, trackingValueRef, valueClassInfoWriteCache); generics.popGenericType(); mapChunkWriter.increaseChunkSize(); } + mapChunkWriter.writeHeader(buffer); + } + + private void javaValueTypeFinalChunkWrite( + Fury fury, + MemoryBuffer buffer, + Map map, + GenericType keyGenericType, + GenericType valueGenericType, + Generics generics) { + MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); + ClassResolver classResolver = fury.getClassResolver(); + RefResolver refResolver = fury.getRefResolver(); + Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); + boolean trackingKeyRef = fury.getClassResolver().needToWriteRef(keyGenericType.getCls()); + for (Object object : map.entrySet()) { + Map.Entry entry = (Map.Entry) object; + Object key = entry.getKey(); + Object value = entry.getValue(); + mapChunkWriter = mapChunkWriter.next(key, value, buffer); + generics.pushGenericType(keyGenericType); + mapChunkWriter.writeKey(key, buffer, classResolver, refResolver, trackingKeyRef, keyClassInfoWriteCache); + generics.popGenericType(); + generics.pushGenericType(valueGenericType); + mapChunkWriter.writeFinalValue(value, buffer, valueSerializer); + generics.popGenericType(); + mapChunkWriter.increaseChunkSize(); + } + mapChunkWriter.writeHeader(buffer); + } + + private void javaKVTypesNonFinalChunkWrite( + Fury fury, + MemoryBuffer buffer, + Map map, + GenericType keyGenericType, + GenericType valueGenericType, + Generics generics) { + MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); + ClassResolver classResolver = fury.getClassResolver(); + RefResolver refResolver = fury.getRefResolver(); + boolean trackingKeyRef = fury.getClassResolver().needToWriteRef(keyGenericType.getCls()); + boolean trackingValueRef = fury.getClassResolver().needToWriteRef(valueGenericType.getCls()); + for (Object object : map.entrySet()) { + Map.Entry entry = (Map.Entry) object; + Object key = entry.getKey(); + Object value = entry.getValue(); + mapChunkWriter = mapChunkWriter.next(key, value, buffer); + if (mapChunkWriter.isMarkChunkWriteFinish()) { + writeJavaRefOptimized(fury, classResolver, refResolver, trackingKeyRef, buffer, key, keyClassInfoWriteCache); + writeJavaRefOptimized(fury, classResolver, refResolver, trackingValueRef, buffer, value, keyClassInfoWriteCache); + } else { + generics.pushGenericType(keyGenericType); + mapChunkWriter.writeKey(key, buffer, classResolver, refResolver, trackingKeyRef, keyClassInfoWriteCache); + generics.popGenericType(); + generics.pushGenericType(valueGenericType); + mapChunkWriter.writeValue(value, buffer, classResolver, refResolver, trackingValueRef, valueClassInfoWriteCache); + generics.popGenericType(); + mapChunkWriter.increaseChunkSize(); + } + } + mapChunkWriter.writeHeader(buffer); + } + + private void javaValueTypeFinalWrite( + Fury fury, + MemoryBuffer buffer, + Map map, + GenericType keyGenericType, + GenericType valueGenericType, + Generics generics) { + ClassResolver classResolver = fury.getClassResolver(); + RefResolver refResolver = fury.getRefResolver(); + boolean trackingKeyRef = fury.getClassResolver().needToWriteRef(keyGenericType.getCls()); + Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); + for (Object object : map.entrySet()) { + Map.Entry entry = (Map.Entry) object; + generics.pushGenericType(keyGenericType); + writeJavaRefOptimized( + fury, + classResolver, + refResolver, + trackingKeyRef, + buffer, + entry.getKey(), + keyClassInfoWriteCache); + generics.popGenericType(); + generics.pushGenericType(valueGenericType); + fury.writeRef(buffer, entry.getValue(), valueSerializer); + generics.popGenericType(); + } + } + + private void javaKVTypesNonFinalWrite( + Fury fury, + MemoryBuffer buffer, + Map map, + GenericType keyGenericType, + GenericType valueGenericType, + Generics generics) { + ClassResolver classResolver = fury.getClassResolver(); + RefResolver refResolver = fury.getRefResolver(); + boolean trackingKeyRef = fury.getClassResolver().needToWriteRef(keyGenericType.getCls()); + boolean trackingValueRef = fury.getClassResolver().needToWriteRef(valueGenericType.getCls()); + for (Object object : map.entrySet()) { + Map.Entry entry = (Map.Entry) object; + generics.pushGenericType(keyGenericType); + writeJavaRefOptimized( + fury, + classResolver, + refResolver, + trackingKeyRef, + buffer, + entry.getKey(), + keyClassInfoWriteCache); + generics.popGenericType(); + generics.pushGenericType(valueGenericType); + writeJavaRefOptimized( + fury, + classResolver, + refResolver, + trackingValueRef, + buffer, + entry.getValue(), + valueClassInfoWriteCache); + generics.popGenericType(); + } + } + + private void generalJavaWrite(Fury fury, MemoryBuffer buffer, Map map) { + ClassResolver classResolver = fury.getClassResolver(); + RefResolver refResolver = fury.getRefResolver(); + for (Object object : map.entrySet()) { + Map.Entry entry = (Map.Entry) object; + writeJavaRefOptimized( + fury, classResolver, refResolver, buffer, entry.getKey(), keyClassInfoWriteCache); + writeJavaRefOptimized( + fury, classResolver, refResolver, buffer, entry.getValue(), valueClassInfoWriteCache); + } } - mapChunkWriter.writeHeader(buffer); - } protected void generalJavaChunkWrite(Fury fury, MemoryBuffer buffer, Map map) { MapChunkWriter chunkWriter = new MapChunkWriter(fury); @@ -365,230 +580,315 @@ protected void generalJavaChunkWrite(Fury fury, MemoryBuffer buffer, Map map) { } } chunkWriter.writeHeader(buffer); - } - - - public static void xwriteElements(Fury fury, MemoryBuffer buffer, Map value) { - Generics generics = fury.getGenerics(); - GenericType genericType = generics.nextGenericType(); - // TODO(chaokunyang) support map subclass whose key or value generics only are available. - if (genericType == null || genericType.getTypeParametersCount() != 2) { - for (Object object : value.entrySet()) { - Map.Entry entry = (Map.Entry) object; - fury.xwriteRef(buffer, entry.getKey()); - fury.xwriteRef(buffer, entry.getValue()); - } - } else { - // TODO(chaokunyang) use codegen to remove all branches. - GenericType keyGenericType = genericType.getTypeParameter0(); - GenericType valueGenericType = genericType.getTypeParameter1(); - Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); - Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); - if (!keyGenericType.hasGenericParameters() && !valueGenericType.hasGenericParameters()) { - for (Object object : value.entrySet()) { - Map.Entry entry = (Map.Entry) object; - fury.xwriteRefByNullableSerializer(buffer, entry.getKey(), keySerializer); - fury.xwriteRefByNullableSerializer(buffer, entry.getValue(), valueSerializer); - } - } else if (valueGenericType.hasGenericParameters()) { - for (Object object : value.entrySet()) { - Map.Entry entry = (Map.Entry) object; - fury.xwriteRefByNullableSerializer(buffer, entry.getKey(), keySerializer); - generics.pushGenericType(valueGenericType); - fury.xwriteRefByNullableSerializer(buffer, entry.getValue(), valueSerializer); - generics.popGenericType(); - } - } else if (keyGenericType.hasGenericParameters()) { - for (Object object : value.entrySet()) { - Map.Entry entry = (Map.Entry) object; - generics.pushGenericType(keyGenericType); - fury.xwriteRefByNullableSerializer(buffer, entry.getKey(), keySerializer); - generics.popGenericType(); - fury.xwriteRefByNullableSerializer(buffer, entry.getValue(), valueSerializer); - } - } else { - for (Object object : value.entrySet()) { - Map.Entry entry = (Map.Entry) object; - generics.pushGenericType(keyGenericType); - fury.xwriteRefByNullableSerializer(buffer, entry.getKey(), keySerializer); - generics.pushGenericType(valueGenericType); - fury.xwriteRefByNullableSerializer(buffer, entry.getValue(), valueSerializer); - } - } - generics.popGenericType(); - } - } - - private Tuple2 getKVGenericType(GenericType genericType) { - Tuple2 genericTypes = partialGenericKVTypeMap.get(genericType); - if (genericTypes == null) { - TypeRef typeRef = genericType.getTypeRef(); - if (!MAP_TYPE.isSupertypeOf(typeRef)) { - Tuple2 typeTuple = Tuple2.of(objType, objType); - partialGenericKVTypeMap.put(genericType, typeTuple); - return typeTuple; - } - Tuple2, TypeRef> mapKeyValueType = TypeUtils.getMapKeyValueType(typeRef); - genericTypes = - Tuple2.of( - fury.getClassResolver().buildGenericType(mapKeyValueType.f0.getType()), - fury.getClassResolver().buildGenericType(mapKeyValueType.f1.getType())); - partialGenericKVTypeMap.put(genericType, genericTypes); - } - return genericTypes; - } - - @Override - public T xread(MemoryBuffer buffer) { - Map map = newMap(buffer); - xreadElements(fury, buffer, map, numElements); - return onMapRead(map); - } - - @SuppressWarnings("unchecked") - protected final void readElements(MemoryBuffer buffer, int size, Map map) { - Serializer keySerializer = this.keySerializer; - Serializer valueSerializer = this.valueSerializer; - // clear the elemSerializer to avoid conflict if the nested - // serialization has collection field. - // TODO use generics for compatible serializer. - this.keySerializer = null; - this.valueSerializer = null; - if (keySerializer != null && valueSerializer != null) { - javaChunkReadWithKVSerializers(buffer, map, size, keySerializer, valueSerializer); - } else if (keySerializer != null) { - javaChunkReadWithKeySerializer(buffer, map, size, keySerializer); - } else if (valueSerializer != null) { - javaChunkReadWithValueSerializer(buffer, map, size, valueSerializer); - } else { - genericJavaRead(fury, buffer, map, size); - } - } - - private void javaChunkReadWithKeySerializer(MemoryBuffer buffer, Map map, int size, Serializer keySerializer) { - MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); - while (size > 0) { - byte chunkSize = buffer.readByte(); - byte header = buffer.readByte(); - Preconditions.checkArgument(chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); - for (byte i = 0; i < chunkSize; i++) { - Object key; - Object value; - key = mapChunkWriter.readFinalKey(buffer, header, keySerializer); - value = mapChunkWriter.readValue(header, buffer, fury.getClassResolver(), fury.trackingRef(), valueClassInfoReadCache); - map.put(key, value); - size--; - } - } - } - - private void javaChunkReadWithValueSerializer(MemoryBuffer buffer, Map map, int size, Serializer valueSerializer) { - MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); - while (size > 0) { - byte chunkSize = buffer.readByte(); - byte header = buffer.readByte(); - Preconditions.checkArgument(chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); - for (byte i = 0; i < chunkSize; i++) { - Object key; - Object value; - key = mapChunkWriter.readKey(header, buffer, fury.getClassResolver(), fury.trackingRef(), keyClassInfoReadCache); - value = mapChunkWriter.readFinalValue(buffer, header, valueSerializer); - map.put(key, value); - size--; - } - } - } - - private void javaChunkReadWithKVSerializers(MemoryBuffer buffer, Map map, int size, Serializer keySerializer, Serializer valueSerializer) { - MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); - while (size > 0) { - byte chunkSize = buffer.readByte(); - byte header = buffer.readByte(); - Preconditions.checkArgument(chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); - for (byte i = 0; i < chunkSize; i++) { - Object key; - Object value; - key = mapChunkWriter.readFinalKey(buffer, header, keySerializer); - value = mapChunkWriter.readFinalValue(buffer, header, valueSerializer); - map.put(key, value); - size--; - } - } - } - - private void genericJavaRead(Fury fury, MemoryBuffer buffer, Map map, int size) { - Generics generics = fury.getGenerics(); - GenericType genericType = generics.nextGenericType(); - if (genericType == null) { - generalJavaChunkRead(fury, buffer, map, size); - } else { - GenericType keyGenericType = genericType.getTypeParameter0(); - GenericType valueGenericType = genericType.getTypeParameter1(); - if (genericType.getTypeParametersCount() < 2) { - Tuple2 kvGenericType = getKVGenericType(genericType); - if (keyGenericType == objType && valueGenericType == objType) { + } + + + public static void xwriteElements(Fury fury, MemoryBuffer buffer, Map value) { + Generics generics = fury.getGenerics(); + GenericType genericType = generics.nextGenericType(); + // TODO(chaokunyang) support map subclass whose key or value generics only are available. + if (genericType == null || genericType.getTypeParametersCount() != 2) { + for (Object object : value.entrySet()) { + Map.Entry entry = (Map.Entry) object; + fury.xwriteRef(buffer, entry.getKey()); + fury.xwriteRef(buffer, entry.getValue()); + } + } else { + // TODO(chaokunyang) use codegen to remove all branches. + GenericType keyGenericType = genericType.getTypeParameter0(); + GenericType valueGenericType = genericType.getTypeParameter1(); + Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); + Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); + if (!keyGenericType.hasGenericParameters() && !valueGenericType.hasGenericParameters()) { + for (Object object : value.entrySet()) { + Map.Entry entry = (Map.Entry) object; + fury.xwriteRefByNullableSerializer(buffer, entry.getKey(), keySerializer); + fury.xwriteRefByNullableSerializer(buffer, entry.getValue(), valueSerializer); + } + } else if (valueGenericType.hasGenericParameters()) { + for (Object object : value.entrySet()) { + Map.Entry entry = (Map.Entry) object; + fury.xwriteRefByNullableSerializer(buffer, entry.getKey(), keySerializer); + generics.pushGenericType(valueGenericType); + fury.xwriteRefByNullableSerializer(buffer, entry.getValue(), valueSerializer); + generics.popGenericType(); + } + } else if (keyGenericType.hasGenericParameters()) { + for (Object object : value.entrySet()) { + Map.Entry entry = (Map.Entry) object; + generics.pushGenericType(keyGenericType); + fury.xwriteRefByNullableSerializer(buffer, entry.getKey(), keySerializer); + generics.popGenericType(); + fury.xwriteRefByNullableSerializer(buffer, entry.getValue(), valueSerializer); + } + } else { + for (Object object : value.entrySet()) { + Map.Entry entry = (Map.Entry) object; + generics.pushGenericType(keyGenericType); + fury.xwriteRefByNullableSerializer(buffer, entry.getKey(), keySerializer); + generics.pushGenericType(valueGenericType); + fury.xwriteRefByNullableSerializer(buffer, entry.getValue(), valueSerializer); + } + } + generics.popGenericType(); + } + } + + private Tuple2 getKVGenericType(GenericType genericType) { + Tuple2 genericTypes = partialGenericKVTypeMap.get(genericType); + if (genericTypes == null) { + TypeRef typeRef = genericType.getTypeRef(); + if (!MAP_TYPE.isSupertypeOf(typeRef)) { + Tuple2 typeTuple = Tuple2.of(objType, objType); + partialGenericKVTypeMap.put(genericType, typeTuple); + return typeTuple; + } + Tuple2, TypeRef> mapKeyValueType = TypeUtils.getMapKeyValueType(typeRef); + genericTypes = + Tuple2.of( + fury.getClassResolver().buildGenericType(mapKeyValueType.f0.getType()), + fury.getClassResolver().buildGenericType(mapKeyValueType.f1.getType())); + partialGenericKVTypeMap.put(genericType, genericTypes); + } + return genericTypes; + } + + @Override + public T xread(MemoryBuffer buffer) { + Map map = newMap(buffer); + xreadElements(fury, buffer, map, numElements); + return onMapRead(map); + } + + @SuppressWarnings("unchecked") + protected final void chunkReadElements(MemoryBuffer buffer, int size, Map map) { + Serializer keySerializer = this.keySerializer; + Serializer valueSerializer = this.valueSerializer; + // clear the elemSerializer to avoid conflict if the nested + // serialization has collection field. + // TODO use generics for compatible serializer. + this.keySerializer = null; + this.valueSerializer = null; + if (keySerializer != null && valueSerializer != null) { + javaChunkReadWithKVSerializers(buffer, map, size, keySerializer, valueSerializer); + } else if (keySerializer != null) { + javaChunkReadWithKeySerializer(buffer, map, size, keySerializer); + } else if (valueSerializer != null) { + javaChunkReadWithValueSerializer(buffer, map, size, valueSerializer); + } else { + genericJavaChunkRead(fury, buffer, map, size); + } + } + + @SuppressWarnings("unchecked") + protected final void readElements(MemoryBuffer buffer, int size, Map map) { + Serializer keySerializer = this.keySerializer; + Serializer valueSerializer = this.valueSerializer; + // clear the elemSerializer to avoid conflict if the nested + // serialization has collection field. + // TODO use generics for compatible serializer. + this.keySerializer = null; + this.valueSerializer = null; + if (keySerializer != null && valueSerializer != null) { + for (int i = 0; i < size; i++) { + Object key = fury.readRef(buffer, keySerializer); + Object value = fury.readRef(buffer, valueSerializer); + map.put(key, value); + } + } else if (keySerializer != null) { + for (int i = 0; i < size; i++) { + Object key = fury.readRef(buffer, keySerializer); + map.put(key, fury.readRef(buffer, keyClassInfoReadCache)); + } + } else if (valueSerializer != null) { + for (int i = 0; i < size; i++) { + Object key = fury.readRef(buffer); + Object value = fury.readRef(buffer, valueSerializer); + map.put(key, value); + } + } else { + genericJavaRead(fury, buffer, map, size); + } + } + + private void genericJavaRead(Fury fury, MemoryBuffer buffer, Map map, int size) { + Generics generics = fury.getGenerics(); + GenericType genericType = generics.nextGenericType(); + if (genericType == null) { + generalJavaRead(fury, buffer, map, size); + } else { + GenericType keyGenericType = genericType.getTypeParameter0(); + GenericType valueGenericType = genericType.getTypeParameter1(); + if (genericType.getTypeParametersCount() < 2) { + Tuple2 kvGenericType = getKVGenericType(genericType); + if (keyGenericType == objType && valueGenericType == objType) { + generalJavaRead(fury, buffer, map, size); + return; + } + keyGenericType = kvGenericType.f0; + valueGenericType = kvGenericType.f1; + } + boolean keyGenericTypeFinal = keyGenericType.isMonomorphic(); + boolean valueGenericTypeFinal = valueGenericType.isMonomorphic(); + if (keyGenericTypeFinal && valueGenericTypeFinal) { + javaKVTypesFinalRead(fury, buffer, map, keyGenericType, valueGenericType, generics, size); + } else if (keyGenericTypeFinal) { + javaKeyTypeFinalRead(fury, buffer, map, keyGenericType, valueGenericType, generics, size); + } else if (valueGenericTypeFinal) { + javaValueTypeFinalRead(fury, buffer, map, keyGenericType, valueGenericType, generics, size); + } else { + javaKVTypesNonFinalRead( + fury, buffer, map, keyGenericType, valueGenericType, generics, size); + } + generics.popGenericType(); + } + } + + private void javaChunkReadWithKeySerializer(MemoryBuffer buffer, Map map, int size, Serializer keySerializer) { + MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); + while (size > 0) { + byte chunkSize = buffer.readByte(); + byte header = buffer.readByte(); + Preconditions.checkArgument(chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); + for (byte i = 0; i < chunkSize; i++) { + Object key; + Object value; + key = mapChunkWriter.readFinalKey(buffer, header, keySerializer); + value = mapChunkWriter.readValue(header, buffer, fury.getClassResolver(), fury.trackingRef(), valueClassInfoReadCache); + map.put(key, value); + size--; + } + } + } + + private void javaChunkReadWithValueSerializer(MemoryBuffer buffer, Map map, int size, Serializer valueSerializer) { + MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); + while (size > 0) { + byte chunkSize = buffer.readByte(); + byte header = buffer.readByte(); + Preconditions.checkArgument(chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); + for (byte i = 0; i < chunkSize; i++) { + Object key; + Object value; + key = mapChunkWriter.readKey(header, buffer, fury.getClassResolver(), fury.trackingRef(), keyClassInfoReadCache); + value = mapChunkWriter.readFinalValue(buffer, header, valueSerializer); + map.put(key, value); + size--; + } + } + } + + private void javaChunkReadWithKVSerializers(MemoryBuffer buffer, Map map, int size, Serializer keySerializer, Serializer valueSerializer) { + MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); + while (size > 0) { + byte chunkSize = buffer.readByte(); + byte header = buffer.readByte(); + Preconditions.checkArgument(chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); + for (byte i = 0; i < chunkSize; i++) { + Object key; + Object value; + key = mapChunkWriter.readFinalKey(buffer, header, keySerializer); + value = mapChunkWriter.readFinalValue(buffer, header, valueSerializer); + map.put(key, value); + size--; + } + } + } + + private void genericJavaChunkRead(Fury fury, MemoryBuffer buffer, Map map, int size) { + Generics generics = fury.getGenerics(); + GenericType genericType = generics.nextGenericType(); + if (genericType == null) { generalJavaChunkRead(fury, buffer, map, size); - return; - } - keyGenericType = kvGenericType.f0; - valueGenericType = kvGenericType.f1; - } - boolean keyGenericTypeFinal = keyGenericType.isMonomorphic(); - boolean valueGenericTypeFinal = valueGenericType.isMonomorphic(); - if (keyGenericTypeFinal && valueGenericTypeFinal) { - javaKVTypesFinalChunkRead(fury, buffer, map, keyGenericType, valueGenericType, generics, size); - } else if (keyGenericTypeFinal) { - javaKeyTypeFinalChunkRead(fury, buffer, map, keyGenericType, valueGenericType, generics, size); - } else if (valueGenericTypeFinal) { - javaValueTypeFinalChunkRead(fury, buffer, map, keyGenericType, valueGenericType, generics, size); - } else { - javaKVTypesNonFinalChunkRead( - fury, buffer, map, keyGenericType, valueGenericType, generics, size); - } - generics.popGenericType(); - } - } - - private void javaKVTypesFinalChunkRead(Fury fury, - MemoryBuffer buffer, - Map map, - GenericType keyGenericType, - GenericType valueGenericType, - Generics generics, - int size) { - Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); - Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); - MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); - while (size > 0) { - byte chunkSize = buffer.readByte(); - byte header = buffer.readByte(); - mapChunkWriter.setKeySerializer(null); - mapChunkWriter.setValueSerializer(null); - Preconditions.checkArgument(chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); - for (byte i = 0; i < chunkSize; i++) { - Object key; - Object value; - generics.pushGenericType(keyGenericType); - key = mapChunkWriter.readFinalKey(buffer, header, keySerializer); - generics.popGenericType(); - generics.pushGenericType(valueGenericType); - value = mapChunkWriter.readFinalValue(buffer, header, valueSerializer); - generics.popGenericType(); - map.put(key, value); - size--; - } - } - - } + } else { + GenericType keyGenericType = genericType.getTypeParameter0(); + GenericType valueGenericType = genericType.getTypeParameter1(); + if (genericType.getTypeParametersCount() < 2) { + Tuple2 kvGenericType = getKVGenericType(genericType); + if (keyGenericType == objType && valueGenericType == objType) { + generalJavaChunkRead(fury, buffer, map, size); + return; + } + keyGenericType = kvGenericType.f0; + valueGenericType = kvGenericType.f1; + } + boolean keyGenericTypeFinal = keyGenericType.isMonomorphic(); + boolean valueGenericTypeFinal = valueGenericType.isMonomorphic(); + if (keyGenericTypeFinal && valueGenericTypeFinal) { + javaKVTypesFinalChunkRead(fury, buffer, map, keyGenericType, valueGenericType, generics, size); + } else if (keyGenericTypeFinal) { + javaKeyTypeFinalChunkRead(fury, buffer, map, keyGenericType, valueGenericType, generics, size); + } else if (valueGenericTypeFinal) { + javaValueTypeFinalChunkRead(fury, buffer, map, keyGenericType, valueGenericType, generics, size); + } else { + javaKVTypesNonFinalChunkRead( + fury, buffer, map, keyGenericType, valueGenericType, generics, size); + } + generics.popGenericType(); + } + } - private void javaKeyTypeFinalChunkRead(Fury fury, + private void javaKVTypesFinalChunkRead(Fury fury, MemoryBuffer buffer, Map map, GenericType keyGenericType, GenericType valueGenericType, Generics generics, int size) { + Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); + Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); + while (size > 0) { + byte chunkSize = buffer.readByte(); + byte header = buffer.readByte(); + mapChunkWriter.setKeySerializer(null); + mapChunkWriter.setValueSerializer(null); + Preconditions.checkArgument(chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); + for (byte i = 0; i < chunkSize; i++) { + Object key; + Object value; + generics.pushGenericType(keyGenericType); + key = mapChunkWriter.readFinalKey(buffer, header, keySerializer); + generics.popGenericType(); + generics.pushGenericType(valueGenericType); + value = mapChunkWriter.readFinalValue(buffer, header, valueSerializer); + generics.popGenericType(); + map.put(key, value); + size--; + } + } + + } + + private void javaKVTypesFinalRead( + Fury fury, + MemoryBuffer buffer, + Map map, + GenericType keyGenericType, + GenericType valueGenericType, + Generics generics, + int size) { + Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); + Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); + for (int i = 0; i < size; i++) { + generics.pushGenericType(keyGenericType); + Object key = fury.readRef(buffer, keySerializer); + generics.popGenericType(); + generics.pushGenericType(valueGenericType); + Object value = fury.readRef(buffer, valueSerializer); + generics.popGenericType(); + map.put(key, value); + } + } + + private void javaKeyTypeFinalChunkRead(Fury fury, + MemoryBuffer buffer, + Map map, + GenericType keyGenericType, + GenericType valueGenericType, + Generics generics, + int size) { + final MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); ClassResolver classResolver = fury.getClassResolver(); boolean trackingValueRef = classResolver.needToWriteRef(valueGenericType.getCls()); Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); @@ -612,299 +912,379 @@ private void javaKeyTypeFinalChunkRead(Fury fury, } } - - private void javaValueTypeFinalChunkRead(Fury fury - , MemoryBuffer buffer - , Map map - , GenericType keyGenericType - , GenericType valueGenericType - , Generics generics - , int size) { - MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); - boolean trackingKeyRef = fury.getClassResolver().needToWriteRef(keyGenericType.getCls()); - Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); - ClassResolver classResolver = fury.getClassResolver(); - while (size > 0) { - byte chunkSize = buffer.readByte(); - Preconditions.checkArgument(chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); - byte header = buffer.readByte(); - mapChunkWriter.setKeySerializer(null); - mapChunkWriter.setValueSerializer(null); - while (chunkSize > 0) { - generics.pushGenericType(keyGenericType); - Object key = mapChunkWriter.readKey(header, buffer, classResolver, trackingKeyRef, keyClassInfoReadCache); - generics.pushGenericType(valueGenericType); - Object value = mapChunkWriter.readFinalValue(buffer, header, valueSerializer); - generics.pushGenericType(valueGenericType); - generics.popGenericType(); - chunkSize--; - size--; - map.put(key, value); - } - } - } - - - private void javaKVTypesNonFinalChunkRead(Fury fury, - MemoryBuffer buffer, - Map map, - GenericType keyGenericType, - GenericType valueGenericType, - Generics generics, - int size) { - final MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); - ClassResolver classResolver = fury.getClassResolver(); - RefResolver refResolver = fury.getRefResolver(); - boolean trackingKeyRef = classResolver.needToWriteRef(keyGenericType.getCls()); - boolean trackingValueRef = classResolver.needToWriteRef(valueGenericType.getCls()); - while (size > 0) { - byte chunkSize = buffer.readByte(); - Preconditions.checkArgument(chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); - if (chunkSize == 0) { - while (size > 0) { - generics.pushGenericType(keyGenericType); - Object key = readJavaRefOptimized(fury, refResolver, trackingKeyRef, buffer, keyClassInfoReadCache); - generics.popGenericType(); - generics.pushGenericType(valueGenericType); - Object value = readJavaRefOptimized(fury, refResolver, trackingValueRef, buffer, valueClassInfoReadCache); - generics.popGenericType(); - map.put(key, value); - size--; - } - } else { - byte header = buffer.readByte(); - mapChunkWriter.setKeySerializer(null); - mapChunkWriter.setValueSerializer(null); - while (chunkSize > 0) { - generics.pushGenericType(keyGenericType); - Object key = mapChunkWriter.readKey(header, buffer, classResolver, trackingKeyRef, keyClassInfoReadCache); - generics.popGenericType(); - generics.pushGenericType(valueGenericType); - Object value = mapChunkWriter.readValue(header, buffer, classResolver, trackingValueRef, valueClassInfoReadCache); - generics.popGenericType(); - chunkSize--; - size--; - map.put(key, value); - } - } - } - } - - - private void generalJavaChunkRead(Fury fury, MemoryBuffer buffer, Map map, int size) { - final MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); - ClassResolver classResolver = fury.getClassResolver(); - boolean trackingKeyRef = fury.trackingRef(); - while (size > 0) { - byte chunkSize = buffer.readByte(); - Preconditions.checkArgument(chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); - if (chunkSize == 0) { - while (size > 0) { - Object key = fury.readRef(buffer, keyClassInfoReadCache); - Object value = fury.readRef(buffer, keyClassInfoReadCache); - map.put(key, value); - size--; - } - } else { - byte header = buffer.readByte(); - mapChunkWriter.setKeySerializer(null); - mapChunkWriter.setValueSerializer(null); - while (chunkSize > 0) { - Object key = mapChunkWriter.readKey(header, buffer, classResolver, trackingKeyRef, keyClassInfoReadCache); - Object value = mapChunkWriter.readValue(header, buffer, classResolver, trackingKeyRef, valueClassInfoReadCache); - chunkSize--; - size--; - map.put(key, value); - } - } - } - } - - @SuppressWarnings("unchecked") - public static void xreadElements(Fury fury, MemoryBuffer buffer, Map map, int size) { - Generics generics = fury.getGenerics(); - GenericType genericType = generics.nextGenericType(); - if (genericType == null || genericType.getTypeParametersCount() != 2) { - for (int i = 0; i < size; i++) { - Object key = fury.xreadRef(buffer); - Object value = fury.xreadRef(buffer); - map.put(key, value); - } - } else { - // TODO(chaokunyang) use codegen to remove all branches. - GenericType keyGenericType = genericType.getTypeParameter0(); - GenericType valueGenericType = genericType.getTypeParameter1(); - Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); - Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); - if (!keyGenericType.hasGenericParameters() && !valueGenericType.hasGenericParameters()) { + private void javaKeyTypeFinalRead(Fury fury + , MemoryBuffer buffer + , Map map + , GenericType keyGenericType + , GenericType valueGenericType + , Generics generics + , int size) { + RefResolver refResolver = fury.getRefResolver(); + boolean trackingValueRef = fury.getClassResolver().needToWriteRef(valueGenericType.getCls()); + Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); for (int i = 0; i < size; i++) { - Object key = fury.xreadRefByNullableSerializer(buffer, keySerializer); - Object value = fury.xreadRefByNullableSerializer(buffer, valueSerializer); - map.put(key, value); + generics.pushGenericType(keyGenericType); + Object key = fury.readRef(buffer, keySerializer); + generics.popGenericType(); + generics.pushGenericType(valueGenericType); + Object value = + readJavaRefOptimized( + fury, refResolver, trackingValueRef, buffer, valueClassInfoWriteCache); + generics.popGenericType(); + map.put(key, value); + } + } + + private void javaValueTypeFinalChunkRead(Fury fury + , MemoryBuffer buffer + , Map map + , GenericType keyGenericType + , GenericType valueGenericType + , Generics generics + , int size) { + MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); + boolean trackingKeyRef = fury.getClassResolver().needToWriteRef(keyGenericType.getCls()); + Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); + ClassResolver classResolver = fury.getClassResolver(); + while (size > 0) { + byte chunkSize = buffer.readByte(); + Preconditions.checkArgument(chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); + byte header = buffer.readByte(); + mapChunkWriter.setKeySerializer(null); + mapChunkWriter.setValueSerializer(null); + while (chunkSize > 0) { + generics.pushGenericType(keyGenericType); + Object key = mapChunkWriter.readKey(header, buffer, classResolver, trackingKeyRef, keyClassInfoReadCache); + generics.pushGenericType(valueGenericType); + Object value = mapChunkWriter.readFinalValue(buffer, header, valueSerializer); + generics.pushGenericType(valueGenericType); + generics.popGenericType(); + chunkSize--; + size--; + map.put(key, value); + } } - } else if (valueGenericType.hasGenericParameters()) { + } + + private void javaValueTypeFinalRead( + Fury fury, + MemoryBuffer buffer, + Map map, + GenericType keyGenericType, + GenericType valueGenericType, + Generics generics, + int size) { + boolean trackingKeyRef = fury.getClassResolver().needToWriteRef(keyGenericType.getCls()); + Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); + RefResolver refResolver = fury.getRefResolver(); for (int i = 0; i < size; i++) { - Object key = fury.xreadRefByNullableSerializer(buffer, keySerializer); - generics.pushGenericType(valueGenericType); - Object value = fury.xreadRefByNullableSerializer(buffer, valueSerializer); - generics.popGenericType(); - map.put(key, value); + generics.pushGenericType(keyGenericType); + Object key = + readJavaRefOptimized(fury, refResolver, trackingKeyRef, buffer, keyClassInfoWriteCache); + generics.popGenericType(); + generics.pushGenericType(valueGenericType); + Object value = fury.readRef(buffer, valueSerializer); + generics.popGenericType(); + map.put(key, value); } - } else if (keyGenericType.hasGenericParameters()) { + } + + private void javaKVTypesNonFinalChunkRead(Fury fury, + MemoryBuffer buffer, + Map map, + GenericType keyGenericType, + GenericType valueGenericType, + Generics generics, + int size) { + final MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); + ClassResolver classResolver = fury.getClassResolver(); + RefResolver refResolver = fury.getRefResolver(); + boolean trackingKeyRef = classResolver.needToWriteRef(keyGenericType.getCls()); + boolean trackingValueRef = classResolver.needToWriteRef(valueGenericType.getCls()); + while (size > 0) { + byte chunkSize = buffer.readByte(); + Preconditions.checkArgument(chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); + if (chunkSize == 0) { + while (size > 0) { + generics.pushGenericType(keyGenericType); + Object key = readJavaRefOptimized(fury, refResolver, trackingKeyRef, buffer, keyClassInfoReadCache); + generics.popGenericType(); + generics.pushGenericType(valueGenericType); + Object value = readJavaRefOptimized(fury, refResolver, trackingValueRef, buffer, valueClassInfoReadCache); + generics.popGenericType(); + map.put(key, value); + size--; + } + } else { + byte header = buffer.readByte(); + mapChunkWriter.setKeySerializer(null); + mapChunkWriter.setValueSerializer(null); + while (chunkSize > 0) { + generics.pushGenericType(keyGenericType); + Object key = mapChunkWriter.readKey(header, buffer, classResolver, trackingKeyRef, keyClassInfoReadCache); + generics.popGenericType(); + generics.pushGenericType(valueGenericType); + Object value = mapChunkWriter.readValue(header, buffer, classResolver, trackingValueRef, valueClassInfoReadCache); + generics.popGenericType(); + chunkSize--; + size--; + map.put(key, value); + } + } + } + } + + private void javaKVTypesNonFinalRead( + Fury fury, + MemoryBuffer buffer, + Map map, + GenericType keyGenericType, + GenericType valueGenericType, + Generics generics, + int size) { + ClassResolver classResolver = fury.getClassResolver(); + RefResolver refResolver = fury.getRefResolver(); + boolean trackingKeyRef = classResolver.needToWriteRef(keyGenericType.getCls()); + boolean trackingValueRef = classResolver.needToWriteRef(valueGenericType.getCls()); for (int i = 0; i < size; i++) { - generics.pushGenericType(keyGenericType); - Object key = fury.xreadRefByNullableSerializer(buffer, keySerializer); - generics.popGenericType(); - Object value = fury.xreadRefByNullableSerializer(buffer, valueSerializer); - map.put(key, value); + generics.pushGenericType(keyGenericType); + Object key = + readJavaRefOptimized(fury, refResolver, trackingKeyRef, buffer, keyClassInfoWriteCache); + generics.popGenericType(); + generics.pushGenericType(valueGenericType); + Object value = + readJavaRefOptimized( + fury, refResolver, trackingValueRef, buffer, valueClassInfoWriteCache); + generics.popGenericType(); + map.put(key, value); + } + } + + + private void generalJavaChunkRead(Fury fury, MemoryBuffer buffer, Map map, int size) { + final MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); + ClassResolver classResolver = fury.getClassResolver(); + boolean trackingKeyRef = fury.trackingRef(); + while (size > 0) { + byte chunkSize = buffer.readByte(); + Preconditions.checkArgument(chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); + if (chunkSize == 0) { + while (size > 0) { + Object key = fury.readRef(buffer, keyClassInfoReadCache); + Object value = fury.readRef(buffer, keyClassInfoReadCache); + map.put(key, value); + size--; + } + } else { + byte header = buffer.readByte(); + mapChunkWriter.setKeySerializer(null); + mapChunkWriter.setValueSerializer(null); + while (chunkSize > 0) { + Object key = mapChunkWriter.readKey(header, buffer, classResolver, trackingKeyRef, keyClassInfoReadCache); + Object value = mapChunkWriter.readValue(header, buffer, classResolver, trackingKeyRef, valueClassInfoReadCache); + chunkSize--; + size--; + map.put(key, value); + } + } } - } else { + } + + private void generalJavaRead(Fury fury, MemoryBuffer buffer, Map map, int size) { for (int i = 0; i < size; i++) { - // FIXME(chaokunyang) nested generics may be get by mistake. - generics.pushGenericType(keyGenericType); - Object key = fury.xreadRefByNullableSerializer(buffer, keySerializer); - generics.pushGenericType(valueGenericType); - Object value = fury.xreadRefByNullableSerializer(buffer, valueSerializer); - map.put(key, value); - } - } - generics.popGenericType(); - } - } - - /** - * Hook for java serialization codegen, read/write key/value by entrySet. - * - *

For key/value type which is final, using codegen may get a big performance gain - * - * @return true if read/write key/value support calling entrySet method - */ - public final boolean supportCodegenHook() { - return supportCodegenHook; - } - - /** - * Write data except size and elements. - * - *

    - * In codegen, follows is call order: - *
  1. write map class if not final - *
  2. write map size - *
  3. onCollectionWrite - *
  4. write keys/values - *
- */ - public abstract Map onMapWrite(MemoryBuffer buffer, T value); - - /** Check null first to avoid ref tracking for some types with ref tracking disabled. */ - private void writeJavaRefOptimized( - Fury fury, - ClassResolver classResolver, - RefResolver refResolver, - MemoryBuffer buffer, - Object obj, - ClassInfoHolder classInfoHolder) { - if (!refResolver.writeNullFlag(buffer, obj)) { - fury.writeRef(buffer, obj, classResolver.getClassInfo(obj.getClass(), classInfoHolder)); - } - } - - private void writeJavaRefOptimized( - Fury fury, - ClassResolver classResolver, - RefResolver refResolver, - boolean trackingRef, - MemoryBuffer buffer, - Object obj, - ClassInfoHolder classInfoHolder) { - if (trackingRef) { - if (!refResolver.writeNullFlag(buffer, obj)) { - fury.writeRef(buffer, obj, classResolver.getClassInfo(obj.getClass(), classInfoHolder)); - } - } else { - if (obj == null) { - buffer.writeByte(Fury.NULL_FLAG); - } else { - buffer.writeByte(Fury.NOT_NULL_VALUE_FLAG); - fury.writeNonRef(buffer, obj, classResolver.getClassInfo(obj.getClass(), classInfoHolder)); - } - } - } - - @Override - public abstract T read(MemoryBuffer buffer); - - /** - * Read data except size and elements, return empty map to be filled. - * - *
    - * In codegen, follows is call order: - *
  1. read map class if not final - *
  2. newMap: read and set map size, read map header and create map. - *
  3. read keys/values - *
- * - *

Map must have default constructor to be invoked by fury, otherwise created object can't be - * used to adding elements. For example: - * - * 

{@code new ArrayList {add(1);}}
- * - *

without default constructor, created list will have elementData as null, adding elements - * will raise NPE. - */ - public Map newMap(MemoryBuffer buffer) { - numElements = buffer.readVarUint32Small7(); - if (constructor == null) { - constructor = ReflectionUtils.getCtrHandle(type, true); - } - try { - Map instance = (Map) constructor.invoke(); - fury.getRefResolver().reference(instance); - return instance; - } catch (Throwable e) { - throw new IllegalArgumentException( - "Please provide public no arguments constructor for class " + type, e); - } - } - - /** - * Get and reset numElements of deserializing collection. Should be called after {@link #newMap}. - * Nested read may overwrite this element, reset is necessary to avoid use wrong value by mistake. - */ - public int getAndClearNumElements() { - int size = numElements; - numElements = -1; // nested read may overwrite this element. - return size; - } - - public void setNumElements(int numElements) { - this.numElements = numElements; - } - - public abstract T onMapRead(Map map); - - private Object readJavaRefOptimized( - Fury fury, - RefResolver refResolver, - boolean trackingRef, - MemoryBuffer buffer, - ClassInfoHolder classInfoHolder) { - if (trackingRef) { - int nextReadRefId = refResolver.tryPreserveRefId(buffer); - if (nextReadRefId >= Fury.NOT_NULL_VALUE_FLAG) { - Object obj = fury.readNonRef(buffer, classInfoHolder); - refResolver.setReadObject(nextReadRefId, obj); - return obj; - } else { - return refResolver.getReadObject(); - } - } else { - byte headFlag = buffer.readByte(); - if (headFlag == Fury.NULL_FLAG) { - return null; - } else { - return fury.readNonRef(buffer, classInfoHolder); - } - } - } + Object key = fury.readRef(buffer, keyClassInfoReadCache); + Object value = fury.readRef(buffer, valueClassInfoReadCache); + map.put(key, value); + } + } + + @SuppressWarnings("unchecked") + public static void xreadElements(Fury fury, MemoryBuffer buffer, Map map, int size) { + Generics generics = fury.getGenerics(); + GenericType genericType = generics.nextGenericType(); + if (genericType == null || genericType.getTypeParametersCount() != 2) { + for (int i = 0; i < size; i++) { + Object key = fury.xreadRef(buffer); + Object value = fury.xreadRef(buffer); + map.put(key, value); + } + } else { + // TODO(chaokunyang) use codegen to remove all branches. + GenericType keyGenericType = genericType.getTypeParameter0(); + GenericType valueGenericType = genericType.getTypeParameter1(); + Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); + Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); + if (!keyGenericType.hasGenericParameters() && !valueGenericType.hasGenericParameters()) { + for (int i = 0; i < size; i++) { + Object key = fury.xreadRefByNullableSerializer(buffer, keySerializer); + Object value = fury.xreadRefByNullableSerializer(buffer, valueSerializer); + map.put(key, value); + } + } else if (valueGenericType.hasGenericParameters()) { + for (int i = 0; i < size; i++) { + Object key = fury.xreadRefByNullableSerializer(buffer, keySerializer); + generics.pushGenericType(valueGenericType); + Object value = fury.xreadRefByNullableSerializer(buffer, valueSerializer); + generics.popGenericType(); + map.put(key, value); + } + } else if (keyGenericType.hasGenericParameters()) { + for (int i = 0; i < size; i++) { + generics.pushGenericType(keyGenericType); + Object key = fury.xreadRefByNullableSerializer(buffer, keySerializer); + generics.popGenericType(); + Object value = fury.xreadRefByNullableSerializer(buffer, valueSerializer); + map.put(key, value); + } + } else { + for (int i = 0; i < size; i++) { + // FIXME(chaokunyang) nested generics may be get by mistake. + generics.pushGenericType(keyGenericType); + Object key = fury.xreadRefByNullableSerializer(buffer, keySerializer); + generics.pushGenericType(valueGenericType); + Object value = fury.xreadRefByNullableSerializer(buffer, valueSerializer); + map.put(key, value); + } + } + generics.popGenericType(); + } + } + + /** + * Hook for java serialization codegen, read/write key/value by entrySet. + * + *

For key/value type which is final, using codegen may get a big performance gain + * + * @return true if read/write key/value support calling entrySet method + */ + public final boolean supportCodegenHook() { + return supportCodegenHook; + } + + /** + * Write data except size and elements. + * + *

    + * In codegen, follows is call order: + *
  1. write map class if not final + *
  2. write map size + *
  3. onCollectionWrite + *
  4. write keys/values + *
+ */ + public abstract Map onMapWrite(MemoryBuffer buffer, T value); + + /** + * Check null first to avoid ref tracking for some types with ref tracking disabled. + */ + private void writeJavaRefOptimized( + Fury fury, + ClassResolver classResolver, + RefResolver refResolver, + MemoryBuffer buffer, + Object obj, + ClassInfoHolder classInfoHolder) { + if (!refResolver.writeNullFlag(buffer, obj)) { + fury.writeRef(buffer, obj, classResolver.getClassInfo(obj.getClass(), classInfoHolder)); + } + } + + private void writeJavaRefOptimized( + Fury fury, + ClassResolver classResolver, + RefResolver refResolver, + boolean trackingRef, + MemoryBuffer buffer, + Object obj, + ClassInfoHolder classInfoHolder) { + if (trackingRef) { + if (!refResolver.writeNullFlag(buffer, obj)) { + fury.writeRef(buffer, obj, classResolver.getClassInfo(obj.getClass(), classInfoHolder)); + } + } else { + if (obj == null) { + buffer.writeByte(Fury.NULL_FLAG); + } else { + buffer.writeByte(Fury.NOT_NULL_VALUE_FLAG); + fury.writeNonRef(buffer, obj, classResolver.getClassInfo(obj.getClass(), classInfoHolder)); + } + } + } + + @Override + public abstract T read(MemoryBuffer buffer); + + /** + * Read data except size and elements, return empty map to be filled. + * + *
    + * In codegen, follows is call order: + *
  1. read map class if not final + *
  2. newMap: read and set map size, read map header and create map. + *
  3. read keys/values + *
+ * + *

Map must have default constructor to be invoked by fury, otherwise created object can't be + * used to adding elements. For example: + * + * 

{@code new ArrayList {add(1);}}
+ * + *

without default constructor, created list will have elementData as null, adding elements + * will raise NPE. + */ + public Map newMap(MemoryBuffer buffer) { + numElements = buffer.readVarUint32Small7(); + if (constructor == null) { + constructor = ReflectionUtils.getCtrHandle(type, true); + } + try { + Map instance = (Map) constructor.invoke(); + fury.getRefResolver().reference(instance); + return instance; + } catch (Throwable e) { + throw new IllegalArgumentException( + "Please provide public no arguments constructor for class " + type, e); + } + } + + /** + * Get and reset numElements of deserializing collection. Should be called after {@link #newMap}. + * Nested read may overwrite this element, reset is necessary to avoid use wrong value by mistake. + */ + public int getAndClearNumElements() { + int size = numElements; + numElements = -1; // nested read may overwrite this element. + return size; + } + + public void setNumElements(int numElements) { + this.numElements = numElements; + } + + public abstract T onMapRead(Map map); + + private Object readJavaRefOptimized( + Fury fury, + RefResolver refResolver, + boolean trackingRef, + MemoryBuffer buffer, + ClassInfoHolder classInfoHolder) { + if (trackingRef) { + int nextReadRefId = refResolver.tryPreserveRefId(buffer); + if (nextReadRefId >= Fury.NOT_NULL_VALUE_FLAG) { + Object obj = fury.readNonRef(buffer, classInfoHolder); + refResolver.setReadObject(nextReadRefId, obj); + return obj; + } else { + return refResolver.getReadObject(); + } + } else { + byte headFlag = buffer.readByte(); + if (headFlag == Fury.NULL_FLAG) { + return null; + } else { + return fury.readNonRef(buffer, classInfoHolder); + } + } + } } diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapSerializer.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapSerializer.java index 0543d5f4e2..bcae76116d 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapSerializer.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapSerializer.java @@ -43,7 +43,11 @@ public Map onMapWrite(MemoryBuffer buffer, T value) { @Override public T read(MemoryBuffer buffer) { Map map = newMap(buffer); - readElements(buffer, getAndClearNumElements(), map); + if (fury.isChunkSerializeMapEnabled()) { + chunkReadElements(buffer, getAndClearNumElements(), map); + } else { + readElements(buffer, getAndClearNumElements(), map); + } return onMapRead(map); } From fe2bb76d0154cac8cbe149e4960d01ab514149a4 Mon Sep 17 00:00:00 2001 From: hening Date: Wed, 31 Jul 2024 23:00:02 +0800 Subject: [PATCH 13/18] mvn spotless apply --- .../src/main/java/org/apache/fury/Fury.java | 12 +- .../java/org/apache/fury/config/Config.java | 1 - .../org/apache/fury/config/FuryBuilder.java | 11 +- .../collection/AbstractMapSerializer.java | 2348 +++++++++-------- .../serializer/collection/MapChunkWriter.java | 943 +++---- .../fury/serializer/collection/MapFlags.java | 53 +- .../serializer/collection/MapSerializer.java | 4 +- .../collection/MapSerializersTest.java | 50 +- 8 files changed, 1755 insertions(+), 1667 deletions(-) diff --git a/java/fury-core/src/main/java/org/apache/fury/Fury.java b/java/fury-core/src/main/java/org/apache/fury/Fury.java index 9e3aa97a53..03fe4c52bd 100644 --- a/java/fury-core/src/main/java/org/apache/fury/Fury.java +++ b/java/fury-core/src/main/java/org/apache/fury/Fury.java @@ -885,12 +885,12 @@ public Object readNullable(MemoryBuffer buffer) { } public Object readNullable(MemoryBuffer buffer, ClassInfoHolder classInfoHolder) { - byte headFlag = buffer.readByte(); - if (headFlag == Fury.NULL_FLAG) { - return null; - } else { - return readNonRef(buffer, classInfoHolder); - } + byte headFlag = buffer.readByte(); + if (headFlag == Fury.NULL_FLAG) { + return null; + } else { + return readNonRef(buffer, classInfoHolder); + } } /** Class should be read already. */ diff --git a/java/fury-core/src/main/java/org/apache/fury/config/Config.java b/java/fury-core/src/main/java/org/apache/fury/config/Config.java index 2b6d261442..c45ecd03e6 100644 --- a/java/fury-core/src/main/java/org/apache/fury/config/Config.java +++ b/java/fury-core/src/main/java/org/apache/fury/config/Config.java @@ -60,7 +60,6 @@ public class Config implements Serializable { private transient int configHash; private final boolean deserializeNonexistentEnumValueAsNull; - public Config(FuryBuilder builder) { language = builder.language; trackingRef = builder.trackingRef; diff --git a/java/fury-core/src/main/java/org/apache/fury/config/FuryBuilder.java b/java/fury-core/src/main/java/org/apache/fury/config/FuryBuilder.java index 44a4b77b29..4d9a302157 100644 --- a/java/fury-core/src/main/java/org/apache/fury/config/FuryBuilder.java +++ b/java/fury-core/src/main/java/org/apache/fury/config/FuryBuilder.java @@ -305,11 +305,12 @@ public FuryBuilder withScalaOptimizationEnabled(boolean enableScalaOptimization) return this; } - /** - * use chunk by chunk method to serialize map, TODO: generate code for chunk by chunk method - * @param chunkSerializeMapEnabled - * @return - */ + /** + * use chunk by chunk method to serialize map, TODO: generate code for chunk by chunk method + * + * @param chunkSerializeMapEnabled + * @return + */ public FuryBuilder withChunkSerializeMapEnable(boolean chunkSerializeMapEnabled) { this.chunkSerializeMapEnabled = chunkSerializeMapEnabled; return this; diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java index 9f422257b7..5e824da958 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java @@ -23,7 +23,6 @@ import java.lang.invoke.MethodHandle; import java.util.Map; - import org.apache.fury.Fury; import org.apache.fury.collection.IdentityMap; import org.apache.fury.collection.Tuple2; @@ -39,1252 +38,1323 @@ import org.apache.fury.type.TypeUtils; import org.apache.fury.util.Preconditions; -/** - * Serializer for all map-like objects. - */ +/** Serializer for all map-like objects. */ @SuppressWarnings({"unchecked", "rawtypes"}) public abstract class AbstractMapSerializer extends Serializer { - protected MethodHandle constructor; - protected final boolean supportCodegenHook; - private Serializer keySerializer; - private Serializer valueSerializer; - protected final ClassInfoHolder keyClassInfoWriteCache; - protected final ClassInfoHolder keyClassInfoReadCache; - protected final ClassInfoHolder valueClassInfoWriteCache; - protected final ClassInfoHolder valueClassInfoReadCache; - // support map subclass whose key or value generics only are available, - // or one of types is already instantiated in subclass, ex: `Subclass implements Map` - private final IdentityMap> partialGenericKVTypeMap; - private final GenericType objType = fury.getClassResolver().buildGenericType(Object.class); - // For subclass whose kv type are instantiated already, such as - // `Subclass implements Map`. If declared `Map` doesn't specify - // instantiated kv type, then the serialization will need to write those kv - // types. Although we can extract this generics when creating the serializer, - // we can't do it when jit `Serializer` for some class which contains one of such map - // field. So we will write those extra kv classes to keep protocol consistency between - // interpreter and jit mode although it seems unnecessary. - // With kv header in future, we can write this kv classes only once, the cost won't be too much. - private int numElements; + protected MethodHandle constructor; + protected final boolean supportCodegenHook; + private Serializer keySerializer; + private Serializer valueSerializer; + protected final ClassInfoHolder keyClassInfoWriteCache; + protected final ClassInfoHolder keyClassInfoReadCache; + protected final ClassInfoHolder valueClassInfoWriteCache; + protected final ClassInfoHolder valueClassInfoReadCache; + // support map subclass whose key or value generics only are available, + // or one of types is already instantiated in subclass, ex: `Subclass implements Map` + private final IdentityMap> partialGenericKVTypeMap; + private final GenericType objType = fury.getClassResolver().buildGenericType(Object.class); + // For subclass whose kv type are instantiated already, such as + // `Subclass implements Map`. If declared `Map` doesn't specify + // instantiated kv type, then the serialization will need to write those kv + // types. Although we can extract this generics when creating the serializer, + // we can't do it when jit `Serializer` for some class which contains one of such map + // field. So we will write those extra kv classes to keep protocol consistency between + // interpreter and jit mode although it seems unnecessary. + // With kv header in future, we can write this kv classes only once, the cost won't be too much. + private int numElements; - public AbstractMapSerializer(Fury fury, Class cls) { - this(fury, cls, !ReflectionUtils.isDynamicGeneratedCLass(cls)); - } + public AbstractMapSerializer(Fury fury, Class cls) { + this(fury, cls, !ReflectionUtils.isDynamicGeneratedCLass(cls)); + } - public AbstractMapSerializer(Fury fury, Class cls, boolean supportCodegenHook) { - super(fury, cls); - this.supportCodegenHook = supportCodegenHook; - keyClassInfoWriteCache = fury.getClassResolver().nilClassInfoHolder(); - keyClassInfoReadCache = fury.getClassResolver().nilClassInfoHolder(); - valueClassInfoWriteCache = fury.getClassResolver().nilClassInfoHolder(); - valueClassInfoReadCache = fury.getClassResolver().nilClassInfoHolder(); - partialGenericKVTypeMap = new IdentityMap<>(); - } + public AbstractMapSerializer(Fury fury, Class cls, boolean supportCodegenHook) { + super(fury, cls); + this.supportCodegenHook = supportCodegenHook; + keyClassInfoWriteCache = fury.getClassResolver().nilClassInfoHolder(); + keyClassInfoReadCache = fury.getClassResolver().nilClassInfoHolder(); + valueClassInfoWriteCache = fury.getClassResolver().nilClassInfoHolder(); + valueClassInfoReadCache = fury.getClassResolver().nilClassInfoHolder(); + partialGenericKVTypeMap = new IdentityMap<>(); + } - /** - * Set key serializer for next serialization, the serializer will be cleared when - * next serialization finished. - */ - public void setKeySerializer(Serializer keySerializer) { - this.keySerializer = keySerializer; - } + /** + * Set key serializer for next serialization, the serializer will be cleared when + * next serialization finished. + */ + public void setKeySerializer(Serializer keySerializer) { + this.keySerializer = keySerializer; + } - /** - * Set value serializer for next serialization, the serializer will be cleared when - * next serialization finished. - */ - public void setValueSerializer(Serializer valueSerializer) { - this.valueSerializer = valueSerializer; + /** + * Set value serializer for next serialization, the serializer will be cleared when + * next serialization finished. + */ + public void setValueSerializer(Serializer valueSerializer) { + this.valueSerializer = valueSerializer; + } + + @Override + public void write(MemoryBuffer buffer, T value) { + Map map = onMapWrite(buffer, value); + if (fury.isChunkSerializeMapEnabled()) { + chunkWriteElements(fury, buffer, map); + } else { + writeElements(fury, buffer, map); } + } - @Override - public void write(MemoryBuffer buffer, T value) { - Map map = onMapWrite(buffer, value); - if (fury.isChunkSerializeMapEnabled()) { - chunkWriteElements(fury, buffer, map); - } else { - writeElements(fury, buffer, map); - } + @Override + public void xwrite(MemoryBuffer buffer, T value) { + Map map = onMapWrite(buffer, value); + xwriteElements(fury, buffer, map); + } + + protected final void writeElements(Fury fury, MemoryBuffer buffer, Map map) { + Serializer keySerializer = this.keySerializer; + Serializer valueSerializer = this.valueSerializer; + // clear the elemSerializer to avoid conflict if the nested + // serialization has collection field. + // TODO use generics for compatible serializer. + this.keySerializer = null; + this.valueSerializer = null; + if (keySerializer != null && valueSerializer != null) { + javaWriteWithKVSerializers(fury, buffer, map, keySerializer, valueSerializer); + } else if (keySerializer != null) { + ClassResolver classResolver = fury.getClassResolver(); + RefResolver refResolver = fury.getRefResolver(); + for (Object object : map.entrySet()) { + Map.Entry entry = (Map.Entry) object; + fury.writeRef(buffer, entry.getKey(), keySerializer); + Object value = entry.getValue(); + writeJavaRefOptimized( + fury, classResolver, refResolver, buffer, value, valueClassInfoWriteCache); + } + } else if (valueSerializer != null) { + ClassResolver classResolver = fury.getClassResolver(); + RefResolver refResolver = fury.getRefResolver(); + for (Object object : map.entrySet()) { + Map.Entry entry = (Map.Entry) object; + Object key = entry.getKey(); + writeJavaRefOptimized( + fury, classResolver, refResolver, buffer, key, keyClassInfoWriteCache); + fury.writeRef(buffer, entry.getValue(), valueSerializer); + } + } else { + genericJavaWrite(fury, buffer, map); } + } - @Override - public void xwrite(MemoryBuffer buffer, T value) { - Map map = onMapWrite(buffer, value); - xwriteElements(fury, buffer, map); + protected final void chunkWriteElements(Fury fury, MemoryBuffer buffer, Map map) { + Serializer keySerializer = this.keySerializer; + Serializer valueSerializer = this.valueSerializer; + // clear the elemSerializer to avoid conflict if the nested + // serialization has collection field. + // TODO use generics for compatible serializer. + this.keySerializer = null; + this.valueSerializer = null; + if (keySerializer != null && valueSerializer != null) { + javaWriteWithKVSerializers(fury, buffer, map, keySerializer, valueSerializer); + } else if (keySerializer != null) { + javaWriteWithKeySerializers(map, buffer, keySerializer); + } else if (valueSerializer != null) { + javaWriteWithValueSerializers(map, buffer, valueSerializer); + } else { + genericJavaChunkWrite(fury, buffer, map); } + } - protected final void writeElements(Fury fury, MemoryBuffer buffer, Map map) { - Serializer keySerializer = this.keySerializer; - Serializer valueSerializer = this.valueSerializer; - // clear the elemSerializer to avoid conflict if the nested - // serialization has collection field. - // TODO use generics for compatible serializer. - this.keySerializer = null; - this.valueSerializer = null; - if (keySerializer != null && valueSerializer != null) { - javaWriteWithKVSerializers(fury, buffer, map, keySerializer, valueSerializer); - } else if (keySerializer != null) { - ClassResolver classResolver = fury.getClassResolver(); - RefResolver refResolver = fury.getRefResolver(); - for (Object object : map.entrySet()) { - Map.Entry entry = (Map.Entry) object; - fury.writeRef(buffer, entry.getKey(), keySerializer); - Object value = entry.getValue(); - writeJavaRefOptimized( - fury, classResolver, refResolver, buffer, value, valueClassInfoWriteCache); - } - } else if (valueSerializer != null) { - ClassResolver classResolver = fury.getClassResolver(); - RefResolver refResolver = fury.getRefResolver(); - for (Object object : map.entrySet()) { - Map.Entry entry = (Map.Entry) object; - Object key = entry.getKey(); - writeJavaRefOptimized( - fury, classResolver, refResolver, buffer, key, keyClassInfoWriteCache); - fury.writeRef(buffer, entry.getValue(), valueSerializer); - } - } else { - genericJavaWrite(fury, buffer, map); - } + private void javaWriteWithKeySerializers(Map map, MemoryBuffer buffer, Serializer keySerializer) { + MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); + ClassResolver classResolver = fury.getClassResolver(); + RefResolver refResolver = fury.getRefResolver(); + for (Object object : map.entrySet()) { + Map.Entry entry = (Map.Entry) object; + mapChunkWriter = mapChunkWriter.next(entry.getKey(), entry.getValue(), buffer); + mapChunkWriter.writeFinalKey(entry.getKey(), buffer, keySerializer); + Object value = entry.getValue(); + mapChunkWriter.writeValue( + value, buffer, classResolver, refResolver, fury.trackingRef(), valueClassInfoWriteCache); + mapChunkWriter.increaseChunkSize(); } + mapChunkWriter.writeHeader(buffer); + } - protected final void chunkWriteElements(Fury fury, MemoryBuffer buffer, Map map) { - Serializer keySerializer = this.keySerializer; - Serializer valueSerializer = this.valueSerializer; - // clear the elemSerializer to avoid conflict if the nested - // serialization has collection field. - // TODO use generics for compatible serializer. - this.keySerializer = null; - this.valueSerializer = null; - if (keySerializer != null && valueSerializer != null) { - javaWriteWithKVSerializers(fury, buffer, map, keySerializer, valueSerializer); - } else if (keySerializer != null) { - javaWriteWithKeySerializers(map, buffer, keySerializer); - } else if (valueSerializer != null) { - javaWriteWithValueSerializers(map, buffer, valueSerializer); - } else { - genericJavaChunkWrite(fury, buffer, map); - } + private void javaWriteWithValueSerializers( + Map map, MemoryBuffer buffer, Serializer valueSerializer) { + MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); + ClassResolver classResolver = fury.getClassResolver(); + RefResolver refResolver = fury.getRefResolver(); + for (Object object : map.entrySet()) { + Map.Entry entry = (Map.Entry) object; + Object key = entry.getKey(); + mapChunkWriter = mapChunkWriter.next(entry.getKey(), entry.getValue(), buffer); + mapChunkWriter.writeKey( + key, buffer, classResolver, refResolver, fury.trackingRef(), keyClassInfoWriteCache); + mapChunkWriter.writeFinalValue(entry.getValue(), buffer, valueSerializer); + mapChunkWriter.increaseChunkSize(); } + mapChunkWriter.writeHeader(buffer); + } - private void javaWriteWithKeySerializers(Map map, MemoryBuffer buffer, Serializer keySerializer) { - MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); - ClassResolver classResolver = fury.getClassResolver(); - RefResolver refResolver = fury.getRefResolver(); - for (Object object : map.entrySet()) { - Map.Entry entry = (Map.Entry) object; - mapChunkWriter = mapChunkWriter.next(entry.getKey(), entry.getValue(), buffer); - mapChunkWriter.writeFinalKey(entry.getKey(), buffer, keySerializer); - Object value = entry.getValue(); - mapChunkWriter.writeValue(value, buffer, classResolver, refResolver, fury.trackingRef(), valueClassInfoWriteCache); - mapChunkWriter.increaseChunkSize(); - } - mapChunkWriter.writeHeader(buffer); + private void javaWriteWithKVSerializers( + Fury fury, + MemoryBuffer buffer, + Map map, + Serializer keySerializer, + Serializer valueSerializer) { + MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); + for (Object object : map.entrySet()) { + Map.Entry entry = (Map.Entry) object; + Object key = entry.getKey(); + Object value = entry.getValue(); + mapChunkWriter = mapChunkWriter.next(entry.getKey(), entry.getValue(), buffer); + mapChunkWriter.writeFinalKey(key, buffer, keySerializer); + mapChunkWriter.writeFinalValue(value, buffer, valueSerializer); + mapChunkWriter.increaseChunkSize(); } + mapChunkWriter.writeHeader(buffer); + } - private void javaWriteWithValueSerializers(Map map, MemoryBuffer buffer, Serializer valueSerializer) { - MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); - ClassResolver classResolver = fury.getClassResolver(); - RefResolver refResolver = fury.getRefResolver(); - for (Object object : map.entrySet()) { - Map.Entry entry = (Map.Entry) object; - Object key = entry.getKey(); - mapChunkWriter = mapChunkWriter.next(entry.getKey(), entry.getValue(), buffer); - mapChunkWriter.writeKey(key, buffer, classResolver, refResolver, fury.trackingRef(), keyClassInfoWriteCache); - mapChunkWriter.writeFinalValue(entry.getValue(), buffer, valueSerializer); - mapChunkWriter.increaseChunkSize(); + private void genericJavaWrite(Fury fury, MemoryBuffer buffer, Map map) { + Generics generics = fury.getGenerics(); + GenericType genericType = generics.nextGenericType(); + if (genericType == null) { + generalJavaWrite(fury, buffer, map); + } else { + GenericType keyGenericType = genericType.getTypeParameter0(); + GenericType valueGenericType = genericType.getTypeParameter1(); + // type parameters count for `Map field` will be 0; + // type parameters count for `SubMap field` which SubMap is + // `SubMap implements Map` will be 1; + if (genericType.getTypeParametersCount() < 2) { + Tuple2 kvGenericType = getKVGenericType(genericType); + if (keyGenericType == objType && valueGenericType == objType) { + generalJavaWrite(fury, buffer, map); + return; } - mapChunkWriter.writeHeader(buffer); + keyGenericType = kvGenericType.f0; + valueGenericType = kvGenericType.f1; + } + // Can't avoid push generics repeatedly in loop by stack depth, because push two + // generic type changed generics stack top, which is depth index, update stack top + // and depth will have some cost too. + // Stack depth to avoid push generics repeatedly in loop. + // Note push two generic type changed generics stack top, which is depth index, + // stack top should be updated when using for serialization k/v. + // int depth = fury.getDepth(); + // // depth + 1 to leave a slot for value generics, otherwise value generics will + // // be overwritten by nested key generics. + // fury.setDepth(depth + 1); + // generics.pushGenericType(keyGenericType); + // fury.setDepth(depth); + // generics.pushGenericType(valueGenericType); + boolean keyGenericTypeFinal = keyGenericType.isMonomorphic(); + boolean valueGenericTypeFinal = valueGenericType.isMonomorphic(); + if (keyGenericTypeFinal && valueGenericTypeFinal) { + javaKVTypesFinalWrite(fury, buffer, map, keyGenericType, valueGenericType, generics); + } else if (keyGenericTypeFinal) { + javaKeyTypeFinalWrite(fury, buffer, map, keyGenericType, valueGenericType, generics); + } else if (valueGenericTypeFinal) { + javaValueTypeFinalWrite(fury, buffer, map, keyGenericType, valueGenericType, generics); + } else { + javaKVTypesNonFinalWrite(fury, buffer, map, keyGenericType, valueGenericType, generics); + } } + } - private void javaWriteWithKVSerializers( - Fury fury, - MemoryBuffer buffer, - Map map, - Serializer keySerializer, - Serializer valueSerializer) { - MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); - for (Object object : map.entrySet()) { - Map.Entry entry = (Map.Entry) object; - Object key = entry.getKey(); - Object value = entry.getValue(); - mapChunkWriter = mapChunkWriter.next(entry.getKey(), entry.getValue(), buffer); - mapChunkWriter.writeFinalKey(key, buffer, keySerializer); - mapChunkWriter.writeFinalValue(value, buffer, valueSerializer); - mapChunkWriter.increaseChunkSize(); + private void genericJavaChunkWrite(Fury fury, MemoryBuffer buffer, Map map) { + Generics generics = fury.getGenerics(); + GenericType genericType = generics.nextGenericType(); + if (genericType == null) { + generalJavaChunkWrite(fury, buffer, map); + } else { + GenericType keyGenericType = genericType.getTypeParameter0(); + GenericType valueGenericType = genericType.getTypeParameter1(); + // type parameters count for `Map field` will be 0; + // type parameters count for `SubMap field` which SubMap is + // `SubMap implements Map` will be 1; + if (genericType.getTypeParametersCount() < 2) { + Tuple2 kvGenericType = getKVGenericType(genericType); + if (keyGenericType == objType && valueGenericType == objType) { + generalJavaChunkWrite(fury, buffer, map); + return; } - mapChunkWriter.writeHeader(buffer); + keyGenericType = kvGenericType.f0; + valueGenericType = kvGenericType.f1; + } + // Can't avoid push generics repeatedly in loop by stack depth, because push two + // generic type changed generics stack top, which is depth index, update stack top + // and depth will have some cost too. + // Stack depth to avoid push generics repeatedly in loop. + // Note push two generic type changed generics stack top, which is depth index, + // stack top should be updated when using for serialization k/v. + // int depth = fury.getDepth(); + // // depth + 1 to leave a slot for value generics, otherwise value generics will + // // be overwritten by nested key generics. + // fury.setDepth(depth + 1); + // generics.pushGenericType(keyGenericType); + // fury.setDepth(depth); + // generics.pushGenericType(valueGenericType); + boolean keyGenericTypeFinal = keyGenericType.isMonomorphic(); + boolean valueGenericTypeFinal = valueGenericType.isMonomorphic(); + if (keyGenericTypeFinal && valueGenericTypeFinal) { + javaKVTypesFinalChunkWrite(fury, buffer, map, keyGenericType, valueGenericType, generics); + } else if (keyGenericTypeFinal) { + javaKeyTypeFinalChunkWrite(fury, buffer, map, keyGenericType, valueGenericType, generics); + } else if (valueGenericTypeFinal) { + javaValueTypeFinalChunkWrite(fury, buffer, map, keyGenericType, valueGenericType, generics); + } else { + javaKVTypesNonFinalChunkWrite( + fury, buffer, map, keyGenericType, valueGenericType, generics); + } } + } - private void genericJavaWrite(Fury fury, MemoryBuffer buffer, Map map) { - Generics generics = fury.getGenerics(); - GenericType genericType = generics.nextGenericType(); - if (genericType == null) { - generalJavaWrite(fury, buffer, map); - } else { - GenericType keyGenericType = genericType.getTypeParameter0(); - GenericType valueGenericType = genericType.getTypeParameter1(); - // type parameters count for `Map field` will be 0; - // type parameters count for `SubMap field` which SubMap is - // `SubMap implements Map` will be 1; - if (genericType.getTypeParametersCount() < 2) { - Tuple2 kvGenericType = getKVGenericType(genericType); - if (keyGenericType == objType && valueGenericType == objType) { - generalJavaWrite(fury, buffer, map); - return; - } - keyGenericType = kvGenericType.f0; - valueGenericType = kvGenericType.f1; - } - // Can't avoid push generics repeatedly in loop by stack depth, because push two - // generic type changed generics stack top, which is depth index, update stack top - // and depth will have some cost too. - // Stack depth to avoid push generics repeatedly in loop. - // Note push two generic type changed generics stack top, which is depth index, - // stack top should be updated when using for serialization k/v. - // int depth = fury.getDepth(); - // // depth + 1 to leave a slot for value generics, otherwise value generics will - // // be overwritten by nested key generics. - // fury.setDepth(depth + 1); - // generics.pushGenericType(keyGenericType); - // fury.setDepth(depth); - // generics.pushGenericType(valueGenericType); - boolean keyGenericTypeFinal = keyGenericType.isMonomorphic(); - boolean valueGenericTypeFinal = valueGenericType.isMonomorphic(); - if (keyGenericTypeFinal && valueGenericTypeFinal) { - javaKVTypesFinalWrite(fury, buffer, map, keyGenericType, valueGenericType, generics); - } else if (keyGenericTypeFinal) { - javaKeyTypeFinalWrite(fury, buffer, map, keyGenericType, valueGenericType, generics); - } else if (valueGenericTypeFinal) { - javaValueTypeFinalWrite(fury, buffer, map, keyGenericType, valueGenericType, generics); - } else { - javaKVTypesNonFinalWrite(fury, buffer, map, keyGenericType, valueGenericType, generics); - } - } + private void javaKVTypesFinalWrite( + Fury fury, + MemoryBuffer buffer, + Map map, + GenericType keyGenericType, + GenericType valueGenericType, + Generics generics) { + Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); + Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); + for (Object object : map.entrySet()) { + Map.Entry entry = (Map.Entry) object; + generics.pushGenericType(keyGenericType); + fury.writeRef(buffer, entry.getKey(), keySerializer); + generics.popGenericType(); + generics.pushGenericType(valueGenericType); + fury.writeRef(buffer, entry.getValue(), valueSerializer); + generics.popGenericType(); } + } - private void genericJavaChunkWrite(Fury fury, MemoryBuffer buffer, Map map) { - Generics generics = fury.getGenerics(); - GenericType genericType = generics.nextGenericType(); - if (genericType == null) { - generalJavaChunkWrite(fury, buffer, map); - } else { - GenericType keyGenericType = genericType.getTypeParameter0(); - GenericType valueGenericType = genericType.getTypeParameter1(); - // type parameters count for `Map field` will be 0; - // type parameters count for `SubMap field` which SubMap is - // `SubMap implements Map` will be 1; - if (genericType.getTypeParametersCount() < 2) { - Tuple2 kvGenericType = getKVGenericType(genericType); - if (keyGenericType == objType && valueGenericType == objType) { - generalJavaChunkWrite(fury, buffer, map); - return; - } - keyGenericType = kvGenericType.f0; - valueGenericType = kvGenericType.f1; - } - // Can't avoid push generics repeatedly in loop by stack depth, because push two - // generic type changed generics stack top, which is depth index, update stack top - // and depth will have some cost too. - // Stack depth to avoid push generics repeatedly in loop. - // Note push two generic type changed generics stack top, which is depth index, - // stack top should be updated when using for serialization k/v. - // int depth = fury.getDepth(); - // // depth + 1 to leave a slot for value generics, otherwise value generics will - // // be overwritten by nested key generics. - // fury.setDepth(depth + 1); - // generics.pushGenericType(keyGenericType); - // fury.setDepth(depth); - // generics.pushGenericType(valueGenericType); - boolean keyGenericTypeFinal = keyGenericType.isMonomorphic(); - boolean valueGenericTypeFinal = valueGenericType.isMonomorphic(); - if (keyGenericTypeFinal && valueGenericTypeFinal) { - javaKVTypesFinalChunkWrite(fury, buffer, map, keyGenericType, valueGenericType, generics); - } else if (keyGenericTypeFinal) { - javaKeyTypeFinalChunkWrite(fury, buffer, map, keyGenericType, valueGenericType, generics); - } else if (valueGenericTypeFinal) { - javaValueTypeFinalChunkWrite(fury, buffer, map, keyGenericType, valueGenericType, generics); - } else { - javaKVTypesNonFinalChunkWrite(fury, buffer, map, keyGenericType, valueGenericType, generics); - } - } + /** + * kv final write do not need to predict , since key and value is almost same type unless null + * + * @param fury + * @param buffer + * @param map + * @param keyGenericType + * @param valueGenericType + * @param generics + */ + private void javaKVTypesFinalChunkWrite( + Fury fury, + MemoryBuffer buffer, + Map map, + GenericType keyGenericType, + GenericType valueGenericType, + Generics generics) { + MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); + Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); + Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); + for (Object object : map.entrySet()) { + Map.Entry entry = (Map.Entry) object; + Object key = entry.getKey(); + Object value = entry.getValue(); + mapChunkWriter = mapChunkWriter.next(key, value, buffer); + generics.pushGenericType(keyGenericType); + mapChunkWriter.writeFinalKey(key, buffer, keySerializer); + generics.popGenericType(); + generics.pushGenericType(valueGenericType); + mapChunkWriter.writeFinalValue(value, buffer, valueSerializer); + generics.popGenericType(); + mapChunkWriter.increaseChunkSize(); } + mapChunkWriter.writeHeader(buffer); + } - private void javaKVTypesFinalWrite( - Fury fury, - MemoryBuffer buffer, - Map map, - GenericType keyGenericType, - GenericType valueGenericType, - Generics generics) { - Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); - Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); - for (Object object : map.entrySet()) { - Map.Entry entry = (Map.Entry) object; - generics.pushGenericType(keyGenericType); - fury.writeRef(buffer, entry.getKey(), keySerializer); - generics.popGenericType(); - generics.pushGenericType(valueGenericType); - fury.writeRef(buffer, entry.getValue(), valueSerializer); - generics.popGenericType(); - } + private void javaKeyTypeFinalWrite( + Fury fury, + MemoryBuffer buffer, + Map map, + GenericType keyGenericType, + GenericType valueGenericType, + Generics generics) { + ClassResolver classResolver = fury.getClassResolver(); + RefResolver refResolver = fury.getRefResolver(); + boolean trackingValueRef = fury.getClassResolver().needToWriteRef(valueGenericType.getCls()); + Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); + for (Object object : map.entrySet()) { + Map.Entry entry = (Map.Entry) object; + generics.pushGenericType(keyGenericType); + fury.writeRef(buffer, entry.getKey(), keySerializer); + generics.popGenericType(); + generics.pushGenericType(valueGenericType); + writeJavaRefOptimized( + fury, + classResolver, + refResolver, + trackingValueRef, + buffer, + entry.getValue(), + valueClassInfoWriteCache); + generics.popGenericType(); } + } - /** - * kv final write do not need to predict , since key and value is almost same type unless null - * - * @param fury - * @param buffer - * @param map - * @param keyGenericType - * @param valueGenericType - * @param generics - */ - private void javaKVTypesFinalChunkWrite( - Fury fury, - MemoryBuffer buffer, - Map map, - GenericType keyGenericType, - GenericType valueGenericType, - Generics generics) { - MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); - Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); - Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); - for (Object object : map.entrySet()) { - Map.Entry entry = (Map.Entry) object; - Object key = entry.getKey(); - Object value = entry.getValue(); - mapChunkWriter = mapChunkWriter.next(key, value, buffer); - generics.pushGenericType(keyGenericType); - mapChunkWriter.writeFinalKey(key, buffer, keySerializer); - generics.popGenericType(); - generics.pushGenericType(valueGenericType); - mapChunkWriter.writeFinalValue(value, buffer, valueSerializer); - generics.popGenericType(); - mapChunkWriter.increaseChunkSize(); - } - mapChunkWriter.writeHeader(buffer); + private void javaKeyTypeFinalChunkWrite( + Fury fury, + MemoryBuffer buffer, + Map map, + GenericType keyGenericType, + GenericType valueGenericType, + Generics generics) { + MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); + ClassResolver classResolver = fury.getClassResolver(); + RefResolver refResolver = fury.getRefResolver(); + Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); + boolean trackingValueRef = fury.getClassResolver().needToWriteRef(valueGenericType.getCls()); + for (Object object : map.entrySet()) { + Map.Entry entry = (Map.Entry) object; + Object key = entry.getKey(); + Object value = entry.getValue(); + mapChunkWriter = mapChunkWriter.next(key, value, buffer); + generics.pushGenericType(keyGenericType); + mapChunkWriter.writeFinalKey(key, buffer, keySerializer); + generics.popGenericType(); + generics.pushGenericType(valueGenericType); + mapChunkWriter.writeValue( + value, buffer, classResolver, refResolver, trackingValueRef, valueClassInfoWriteCache); + generics.popGenericType(); + mapChunkWriter.increaseChunkSize(); } + mapChunkWriter.writeHeader(buffer); + } - private void javaKeyTypeFinalWrite( - Fury fury, - MemoryBuffer buffer, - Map map, - GenericType keyGenericType, - GenericType valueGenericType, - Generics generics) { - ClassResolver classResolver = fury.getClassResolver(); - RefResolver refResolver = fury.getRefResolver(); - boolean trackingValueRef = fury.getClassResolver().needToWriteRef(valueGenericType.getCls()); - Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); - for (Object object : map.entrySet()) { - Map.Entry entry = (Map.Entry) object; - generics.pushGenericType(keyGenericType); - fury.writeRef(buffer, entry.getKey(), keySerializer); - generics.popGenericType(); - generics.pushGenericType(valueGenericType); - writeJavaRefOptimized( - fury, - classResolver, - refResolver, - trackingValueRef, - buffer, - entry.getValue(), - valueClassInfoWriteCache); - generics.popGenericType(); - } + private void javaValueTypeFinalChunkWrite( + Fury fury, + MemoryBuffer buffer, + Map map, + GenericType keyGenericType, + GenericType valueGenericType, + Generics generics) { + MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); + ClassResolver classResolver = fury.getClassResolver(); + RefResolver refResolver = fury.getRefResolver(); + Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); + boolean trackingKeyRef = fury.getClassResolver().needToWriteRef(keyGenericType.getCls()); + for (Object object : map.entrySet()) { + Map.Entry entry = (Map.Entry) object; + Object key = entry.getKey(); + Object value = entry.getValue(); + mapChunkWriter = mapChunkWriter.next(key, value, buffer); + generics.pushGenericType(keyGenericType); + mapChunkWriter.writeKey( + key, buffer, classResolver, refResolver, trackingKeyRef, keyClassInfoWriteCache); + generics.popGenericType(); + generics.pushGenericType(valueGenericType); + mapChunkWriter.writeFinalValue(value, buffer, valueSerializer); + generics.popGenericType(); + mapChunkWriter.increaseChunkSize(); } + mapChunkWriter.writeHeader(buffer); + } - private void javaKeyTypeFinalChunkWrite( - Fury fury, - MemoryBuffer buffer, - Map map, - GenericType keyGenericType, - GenericType valueGenericType, - Generics generics) { - MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); - ClassResolver classResolver = fury.getClassResolver(); - RefResolver refResolver = fury.getRefResolver(); - Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); - boolean trackingValueRef = fury.getClassResolver().needToWriteRef(valueGenericType.getCls()); - for (Object object : map.entrySet()) { - Map.Entry entry = (Map.Entry) object; - Object key = entry.getKey(); - Object value = entry.getValue(); - mapChunkWriter = mapChunkWriter.next(key, value, buffer); - generics.pushGenericType(keyGenericType); - mapChunkWriter.writeFinalKey(key, buffer, keySerializer); - generics.popGenericType(); - generics.pushGenericType(valueGenericType); - mapChunkWriter.writeValue(value, buffer, classResolver, refResolver, trackingValueRef, valueClassInfoWriteCache); - generics.popGenericType(); - mapChunkWriter.increaseChunkSize(); - } - mapChunkWriter.writeHeader(buffer); + private void javaKVTypesNonFinalChunkWrite( + Fury fury, + MemoryBuffer buffer, + Map map, + GenericType keyGenericType, + GenericType valueGenericType, + Generics generics) { + MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); + ClassResolver classResolver = fury.getClassResolver(); + RefResolver refResolver = fury.getRefResolver(); + boolean trackingKeyRef = fury.getClassResolver().needToWriteRef(keyGenericType.getCls()); + boolean trackingValueRef = fury.getClassResolver().needToWriteRef(valueGenericType.getCls()); + for (Object object : map.entrySet()) { + Map.Entry entry = (Map.Entry) object; + Object key = entry.getKey(); + Object value = entry.getValue(); + mapChunkWriter = mapChunkWriter.next(key, value, buffer); + if (mapChunkWriter.isMarkChunkWriteFinish()) { + writeJavaRefOptimized( + fury, classResolver, refResolver, trackingKeyRef, buffer, key, keyClassInfoWriteCache); + writeJavaRefOptimized( + fury, + classResolver, + refResolver, + trackingValueRef, + buffer, + value, + keyClassInfoWriteCache); + } else { + generics.pushGenericType(keyGenericType); + mapChunkWriter.writeKey( + key, buffer, classResolver, refResolver, trackingKeyRef, keyClassInfoWriteCache); + generics.popGenericType(); + generics.pushGenericType(valueGenericType); + mapChunkWriter.writeValue( + value, buffer, classResolver, refResolver, trackingValueRef, valueClassInfoWriteCache); + generics.popGenericType(); + mapChunkWriter.increaseChunkSize(); + } } + mapChunkWriter.writeHeader(buffer); + } - private void javaValueTypeFinalChunkWrite( - Fury fury, - MemoryBuffer buffer, - Map map, - GenericType keyGenericType, - GenericType valueGenericType, - Generics generics) { - MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); - ClassResolver classResolver = fury.getClassResolver(); - RefResolver refResolver = fury.getRefResolver(); - Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); - boolean trackingKeyRef = fury.getClassResolver().needToWriteRef(keyGenericType.getCls()); - for (Object object : map.entrySet()) { - Map.Entry entry = (Map.Entry) object; - Object key = entry.getKey(); - Object value = entry.getValue(); - mapChunkWriter = mapChunkWriter.next(key, value, buffer); - generics.pushGenericType(keyGenericType); - mapChunkWriter.writeKey(key, buffer, classResolver, refResolver, trackingKeyRef, keyClassInfoWriteCache); - generics.popGenericType(); - generics.pushGenericType(valueGenericType); - mapChunkWriter.writeFinalValue(value, buffer, valueSerializer); - generics.popGenericType(); - mapChunkWriter.increaseChunkSize(); - } - mapChunkWriter.writeHeader(buffer); + private void javaValueTypeFinalWrite( + Fury fury, + MemoryBuffer buffer, + Map map, + GenericType keyGenericType, + GenericType valueGenericType, + Generics generics) { + ClassResolver classResolver = fury.getClassResolver(); + RefResolver refResolver = fury.getRefResolver(); + boolean trackingKeyRef = fury.getClassResolver().needToWriteRef(keyGenericType.getCls()); + Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); + for (Object object : map.entrySet()) { + Map.Entry entry = (Map.Entry) object; + generics.pushGenericType(keyGenericType); + writeJavaRefOptimized( + fury, + classResolver, + refResolver, + trackingKeyRef, + buffer, + entry.getKey(), + keyClassInfoWriteCache); + generics.popGenericType(); + generics.pushGenericType(valueGenericType); + fury.writeRef(buffer, entry.getValue(), valueSerializer); + generics.popGenericType(); } + } - private void javaKVTypesNonFinalChunkWrite( - Fury fury, - MemoryBuffer buffer, - Map map, - GenericType keyGenericType, - GenericType valueGenericType, - Generics generics) { - MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); - ClassResolver classResolver = fury.getClassResolver(); - RefResolver refResolver = fury.getRefResolver(); - boolean trackingKeyRef = fury.getClassResolver().needToWriteRef(keyGenericType.getCls()); - boolean trackingValueRef = fury.getClassResolver().needToWriteRef(valueGenericType.getCls()); - for (Object object : map.entrySet()) { - Map.Entry entry = (Map.Entry) object; - Object key = entry.getKey(); - Object value = entry.getValue(); - mapChunkWriter = mapChunkWriter.next(key, value, buffer); - if (mapChunkWriter.isMarkChunkWriteFinish()) { - writeJavaRefOptimized(fury, classResolver, refResolver, trackingKeyRef, buffer, key, keyClassInfoWriteCache); - writeJavaRefOptimized(fury, classResolver, refResolver, trackingValueRef, buffer, value, keyClassInfoWriteCache); - } else { - generics.pushGenericType(keyGenericType); - mapChunkWriter.writeKey(key, buffer, classResolver, refResolver, trackingKeyRef, keyClassInfoWriteCache); - generics.popGenericType(); - generics.pushGenericType(valueGenericType); - mapChunkWriter.writeValue(value, buffer, classResolver, refResolver, trackingValueRef, valueClassInfoWriteCache); - generics.popGenericType(); - mapChunkWriter.increaseChunkSize(); - } - } - mapChunkWriter.writeHeader(buffer); + private void javaKVTypesNonFinalWrite( + Fury fury, + MemoryBuffer buffer, + Map map, + GenericType keyGenericType, + GenericType valueGenericType, + Generics generics) { + ClassResolver classResolver = fury.getClassResolver(); + RefResolver refResolver = fury.getRefResolver(); + boolean trackingKeyRef = fury.getClassResolver().needToWriteRef(keyGenericType.getCls()); + boolean trackingValueRef = fury.getClassResolver().needToWriteRef(valueGenericType.getCls()); + for (Object object : map.entrySet()) { + Map.Entry entry = (Map.Entry) object; + generics.pushGenericType(keyGenericType); + writeJavaRefOptimized( + fury, + classResolver, + refResolver, + trackingKeyRef, + buffer, + entry.getKey(), + keyClassInfoWriteCache); + generics.popGenericType(); + generics.pushGenericType(valueGenericType); + writeJavaRefOptimized( + fury, + classResolver, + refResolver, + trackingValueRef, + buffer, + entry.getValue(), + valueClassInfoWriteCache); + generics.popGenericType(); } + } - private void javaValueTypeFinalWrite( - Fury fury, - MemoryBuffer buffer, - Map map, - GenericType keyGenericType, - GenericType valueGenericType, - Generics generics) { - ClassResolver classResolver = fury.getClassResolver(); - RefResolver refResolver = fury.getRefResolver(); - boolean trackingKeyRef = fury.getClassResolver().needToWriteRef(keyGenericType.getCls()); - Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); - for (Object object : map.entrySet()) { - Map.Entry entry = (Map.Entry) object; - generics.pushGenericType(keyGenericType); - writeJavaRefOptimized( - fury, - classResolver, - refResolver, - trackingKeyRef, - buffer, - entry.getKey(), - keyClassInfoWriteCache); - generics.popGenericType(); - generics.pushGenericType(valueGenericType); - fury.writeRef(buffer, entry.getValue(), valueSerializer); - generics.popGenericType(); - } + private void generalJavaWrite(Fury fury, MemoryBuffer buffer, Map map) { + ClassResolver classResolver = fury.getClassResolver(); + RefResolver refResolver = fury.getRefResolver(); + for (Object object : map.entrySet()) { + Map.Entry entry = (Map.Entry) object; + writeJavaRefOptimized( + fury, classResolver, refResolver, buffer, entry.getKey(), keyClassInfoWriteCache); + writeJavaRefOptimized( + fury, classResolver, refResolver, buffer, entry.getValue(), valueClassInfoWriteCache); } + } - private void javaKVTypesNonFinalWrite( - Fury fury, - MemoryBuffer buffer, - Map map, - GenericType keyGenericType, - GenericType valueGenericType, - Generics generics) { - ClassResolver classResolver = fury.getClassResolver(); - RefResolver refResolver = fury.getRefResolver(); - boolean trackingKeyRef = fury.getClassResolver().needToWriteRef(keyGenericType.getCls()); - boolean trackingValueRef = fury.getClassResolver().needToWriteRef(valueGenericType.getCls()); - for (Object object : map.entrySet()) { - Map.Entry entry = (Map.Entry) object; - generics.pushGenericType(keyGenericType); - writeJavaRefOptimized( - fury, - classResolver, - refResolver, - trackingKeyRef, - buffer, - entry.getKey(), - keyClassInfoWriteCache); - generics.popGenericType(); - generics.pushGenericType(valueGenericType); - writeJavaRefOptimized( - fury, - classResolver, - refResolver, - trackingValueRef, - buffer, - entry.getValue(), - valueClassInfoWriteCache); - generics.popGenericType(); - } + protected void generalJavaChunkWrite(Fury fury, MemoryBuffer buffer, Map map) { + MapChunkWriter chunkWriter = new MapChunkWriter(fury); + ClassResolver classResolver = fury.getClassResolver(); + RefResolver refResolver = fury.getRefResolver(); + for (Object object : map.entrySet()) { + Map.Entry entry = (Map.Entry) object; + Object key = entry.getKey(); + Object value = entry.getValue(); + chunkWriter = chunkWriter.next(key, value, buffer); + if (!chunkWriter.isMarkChunkWriteFinish()) { + chunkWriter.generalChunkWrite( + key, + value, + buffer, + classResolver, + refResolver, + keyClassInfoWriteCache, + valueClassInfoWriteCache); + } else { + writeJavaRefOptimized( + fury, classResolver, refResolver, buffer, entry.getKey(), keyClassInfoWriteCache); + writeJavaRefOptimized( + fury, classResolver, refResolver, buffer, entry.getValue(), valueClassInfoWriteCache); + } } + chunkWriter.writeHeader(buffer); + } - private void generalJavaWrite(Fury fury, MemoryBuffer buffer, Map map) { - ClassResolver classResolver = fury.getClassResolver(); - RefResolver refResolver = fury.getRefResolver(); - for (Object object : map.entrySet()) { - Map.Entry entry = (Map.Entry) object; - writeJavaRefOptimized( - fury, classResolver, refResolver, buffer, entry.getKey(), keyClassInfoWriteCache); - writeJavaRefOptimized( - fury, classResolver, refResolver, buffer, entry.getValue(), valueClassInfoWriteCache); + public static void xwriteElements(Fury fury, MemoryBuffer buffer, Map value) { + Generics generics = fury.getGenerics(); + GenericType genericType = generics.nextGenericType(); + // TODO(chaokunyang) support map subclass whose key or value generics only are available. + if (genericType == null || genericType.getTypeParametersCount() != 2) { + for (Object object : value.entrySet()) { + Map.Entry entry = (Map.Entry) object; + fury.xwriteRef(buffer, entry.getKey()); + fury.xwriteRef(buffer, entry.getValue()); + } + } else { + // TODO(chaokunyang) use codegen to remove all branches. + GenericType keyGenericType = genericType.getTypeParameter0(); + GenericType valueGenericType = genericType.getTypeParameter1(); + Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); + Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); + if (!keyGenericType.hasGenericParameters() && !valueGenericType.hasGenericParameters()) { + for (Object object : value.entrySet()) { + Map.Entry entry = (Map.Entry) object; + fury.xwriteRefByNullableSerializer(buffer, entry.getKey(), keySerializer); + fury.xwriteRefByNullableSerializer(buffer, entry.getValue(), valueSerializer); + } + } else if (valueGenericType.hasGenericParameters()) { + for (Object object : value.entrySet()) { + Map.Entry entry = (Map.Entry) object; + fury.xwriteRefByNullableSerializer(buffer, entry.getKey(), keySerializer); + generics.pushGenericType(valueGenericType); + fury.xwriteRefByNullableSerializer(buffer, entry.getValue(), valueSerializer); + generics.popGenericType(); + } + } else if (keyGenericType.hasGenericParameters()) { + for (Object object : value.entrySet()) { + Map.Entry entry = (Map.Entry) object; + generics.pushGenericType(keyGenericType); + fury.xwriteRefByNullableSerializer(buffer, entry.getKey(), keySerializer); + generics.popGenericType(); + fury.xwriteRefByNullableSerializer(buffer, entry.getValue(), valueSerializer); } + } else { + for (Object object : value.entrySet()) { + Map.Entry entry = (Map.Entry) object; + generics.pushGenericType(keyGenericType); + fury.xwriteRefByNullableSerializer(buffer, entry.getKey(), keySerializer); + generics.pushGenericType(valueGenericType); + fury.xwriteRefByNullableSerializer(buffer, entry.getValue(), valueSerializer); + } + } + generics.popGenericType(); } + } - protected void generalJavaChunkWrite(Fury fury, MemoryBuffer buffer, Map map) { - MapChunkWriter chunkWriter = new MapChunkWriter(fury); - ClassResolver classResolver = fury.getClassResolver(); - RefResolver refResolver = fury.getRefResolver(); - for (Object object : map.entrySet()) { - Map.Entry entry = (Map.Entry) object; - Object key = entry.getKey(); - Object value = entry.getValue(); - chunkWriter = chunkWriter.next(key, value, buffer); - if (!chunkWriter.isMarkChunkWriteFinish()) { - chunkWriter.generalChunkWrite(key, value, buffer, classResolver, refResolver, keyClassInfoWriteCache, valueClassInfoWriteCache); - } else { - writeJavaRefOptimized( - fury, classResolver, refResolver, buffer, entry.getKey(), keyClassInfoWriteCache); - writeJavaRefOptimized( - fury, classResolver, refResolver, buffer, entry.getValue(), valueClassInfoWriteCache); - } - } - chunkWriter.writeHeader(buffer); + private Tuple2 getKVGenericType(GenericType genericType) { + Tuple2 genericTypes = partialGenericKVTypeMap.get(genericType); + if (genericTypes == null) { + TypeRef typeRef = genericType.getTypeRef(); + if (!MAP_TYPE.isSupertypeOf(typeRef)) { + Tuple2 typeTuple = Tuple2.of(objType, objType); + partialGenericKVTypeMap.put(genericType, typeTuple); + return typeTuple; + } + Tuple2, TypeRef> mapKeyValueType = TypeUtils.getMapKeyValueType(typeRef); + genericTypes = + Tuple2.of( + fury.getClassResolver().buildGenericType(mapKeyValueType.f0.getType()), + fury.getClassResolver().buildGenericType(mapKeyValueType.f1.getType())); + partialGenericKVTypeMap.put(genericType, genericTypes); } + return genericTypes; + } + @Override + public T xread(MemoryBuffer buffer) { + Map map = newMap(buffer); + xreadElements(fury, buffer, map, numElements); + return onMapRead(map); + } - public static void xwriteElements(Fury fury, MemoryBuffer buffer, Map value) { - Generics generics = fury.getGenerics(); - GenericType genericType = generics.nextGenericType(); - // TODO(chaokunyang) support map subclass whose key or value generics only are available. - if (genericType == null || genericType.getTypeParametersCount() != 2) { - for (Object object : value.entrySet()) { - Map.Entry entry = (Map.Entry) object; - fury.xwriteRef(buffer, entry.getKey()); - fury.xwriteRef(buffer, entry.getValue()); - } - } else { - // TODO(chaokunyang) use codegen to remove all branches. - GenericType keyGenericType = genericType.getTypeParameter0(); - GenericType valueGenericType = genericType.getTypeParameter1(); - Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); - Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); - if (!keyGenericType.hasGenericParameters() && !valueGenericType.hasGenericParameters()) { - for (Object object : value.entrySet()) { - Map.Entry entry = (Map.Entry) object; - fury.xwriteRefByNullableSerializer(buffer, entry.getKey(), keySerializer); - fury.xwriteRefByNullableSerializer(buffer, entry.getValue(), valueSerializer); - } - } else if (valueGenericType.hasGenericParameters()) { - for (Object object : value.entrySet()) { - Map.Entry entry = (Map.Entry) object; - fury.xwriteRefByNullableSerializer(buffer, entry.getKey(), keySerializer); - generics.pushGenericType(valueGenericType); - fury.xwriteRefByNullableSerializer(buffer, entry.getValue(), valueSerializer); - generics.popGenericType(); - } - } else if (keyGenericType.hasGenericParameters()) { - for (Object object : value.entrySet()) { - Map.Entry entry = (Map.Entry) object; - generics.pushGenericType(keyGenericType); - fury.xwriteRefByNullableSerializer(buffer, entry.getKey(), keySerializer); - generics.popGenericType(); - fury.xwriteRefByNullableSerializer(buffer, entry.getValue(), valueSerializer); - } - } else { - for (Object object : value.entrySet()) { - Map.Entry entry = (Map.Entry) object; - generics.pushGenericType(keyGenericType); - fury.xwriteRefByNullableSerializer(buffer, entry.getKey(), keySerializer); - generics.pushGenericType(valueGenericType); - fury.xwriteRefByNullableSerializer(buffer, entry.getValue(), valueSerializer); - } - } - generics.popGenericType(); - } + @SuppressWarnings("unchecked") + protected final void chunkReadElements(MemoryBuffer buffer, int size, Map map) { + Serializer keySerializer = this.keySerializer; + Serializer valueSerializer = this.valueSerializer; + // clear the elemSerializer to avoid conflict if the nested + // serialization has collection field. + // TODO use generics for compatible serializer. + this.keySerializer = null; + this.valueSerializer = null; + if (keySerializer != null && valueSerializer != null) { + javaChunkReadWithKVSerializers(buffer, map, size, keySerializer, valueSerializer); + } else if (keySerializer != null) { + javaChunkReadWithKeySerializer(buffer, map, size, keySerializer); + } else if (valueSerializer != null) { + javaChunkReadWithValueSerializer(buffer, map, size, valueSerializer); + } else { + genericJavaChunkRead(fury, buffer, map, size); } + } - private Tuple2 getKVGenericType(GenericType genericType) { - Tuple2 genericTypes = partialGenericKVTypeMap.get(genericType); - if (genericTypes == null) { - TypeRef typeRef = genericType.getTypeRef(); - if (!MAP_TYPE.isSupertypeOf(typeRef)) { - Tuple2 typeTuple = Tuple2.of(objType, objType); - partialGenericKVTypeMap.put(genericType, typeTuple); - return typeTuple; - } - Tuple2, TypeRef> mapKeyValueType = TypeUtils.getMapKeyValueType(typeRef); - genericTypes = - Tuple2.of( - fury.getClassResolver().buildGenericType(mapKeyValueType.f0.getType()), - fury.getClassResolver().buildGenericType(mapKeyValueType.f1.getType())); - partialGenericKVTypeMap.put(genericType, genericTypes); - } - return genericTypes; + @SuppressWarnings("unchecked") + protected final void readElements(MemoryBuffer buffer, int size, Map map) { + Serializer keySerializer = this.keySerializer; + Serializer valueSerializer = this.valueSerializer; + // clear the elemSerializer to avoid conflict if the nested + // serialization has collection field. + // TODO use generics for compatible serializer. + this.keySerializer = null; + this.valueSerializer = null; + if (keySerializer != null && valueSerializer != null) { + for (int i = 0; i < size; i++) { + Object key = fury.readRef(buffer, keySerializer); + Object value = fury.readRef(buffer, valueSerializer); + map.put(key, value); + } + } else if (keySerializer != null) { + for (int i = 0; i < size; i++) { + Object key = fury.readRef(buffer, keySerializer); + map.put(key, fury.readRef(buffer, keyClassInfoReadCache)); + } + } else if (valueSerializer != null) { + for (int i = 0; i < size; i++) { + Object key = fury.readRef(buffer); + Object value = fury.readRef(buffer, valueSerializer); + map.put(key, value); + } + } else { + genericJavaRead(fury, buffer, map, size); } + } - @Override - public T xread(MemoryBuffer buffer) { - Map map = newMap(buffer); - xreadElements(fury, buffer, map, numElements); - return onMapRead(map); + private void genericJavaRead(Fury fury, MemoryBuffer buffer, Map map, int size) { + Generics generics = fury.getGenerics(); + GenericType genericType = generics.nextGenericType(); + if (genericType == null) { + generalJavaRead(fury, buffer, map, size); + } else { + GenericType keyGenericType = genericType.getTypeParameter0(); + GenericType valueGenericType = genericType.getTypeParameter1(); + if (genericType.getTypeParametersCount() < 2) { + Tuple2 kvGenericType = getKVGenericType(genericType); + if (keyGenericType == objType && valueGenericType == objType) { + generalJavaRead(fury, buffer, map, size); + return; + } + keyGenericType = kvGenericType.f0; + valueGenericType = kvGenericType.f1; + } + boolean keyGenericTypeFinal = keyGenericType.isMonomorphic(); + boolean valueGenericTypeFinal = valueGenericType.isMonomorphic(); + if (keyGenericTypeFinal && valueGenericTypeFinal) { + javaKVTypesFinalRead(fury, buffer, map, keyGenericType, valueGenericType, generics, size); + } else if (keyGenericTypeFinal) { + javaKeyTypeFinalRead(fury, buffer, map, keyGenericType, valueGenericType, generics, size); + } else if (valueGenericTypeFinal) { + javaValueTypeFinalRead(fury, buffer, map, keyGenericType, valueGenericType, generics, size); + } else { + javaKVTypesNonFinalRead( + fury, buffer, map, keyGenericType, valueGenericType, generics, size); + } + generics.popGenericType(); } + } - @SuppressWarnings("unchecked") - protected final void chunkReadElements(MemoryBuffer buffer, int size, Map map) { - Serializer keySerializer = this.keySerializer; - Serializer valueSerializer = this.valueSerializer; - // clear the elemSerializer to avoid conflict if the nested - // serialization has collection field. - // TODO use generics for compatible serializer. - this.keySerializer = null; - this.valueSerializer = null; - if (keySerializer != null && valueSerializer != null) { - javaChunkReadWithKVSerializers(buffer, map, size, keySerializer, valueSerializer); - } else if (keySerializer != null) { - javaChunkReadWithKeySerializer(buffer, map, size, keySerializer); - } else if (valueSerializer != null) { - javaChunkReadWithValueSerializer(buffer, map, size, valueSerializer); - } else { - genericJavaChunkRead(fury, buffer, map, size); - } + private void javaChunkReadWithKeySerializer( + MemoryBuffer buffer, Map map, int size, Serializer keySerializer) { + MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); + while (size > 0) { + byte chunkSize = buffer.readByte(); + byte header = buffer.readByte(); + Preconditions.checkArgument( + chunkSize >= 0, + "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); + for (byte i = 0; i < chunkSize; i++) { + Object key; + Object value; + key = mapChunkWriter.readFinalKey(buffer, header, keySerializer); + value = + mapChunkWriter.readValue( + header, + buffer, + fury.getClassResolver(), + fury.trackingRef(), + valueClassInfoReadCache); + map.put(key, value); + size--; + } } + } - @SuppressWarnings("unchecked") - protected final void readElements(MemoryBuffer buffer, int size, Map map) { - Serializer keySerializer = this.keySerializer; - Serializer valueSerializer = this.valueSerializer; - // clear the elemSerializer to avoid conflict if the nested - // serialization has collection field. - // TODO use generics for compatible serializer. - this.keySerializer = null; - this.valueSerializer = null; - if (keySerializer != null && valueSerializer != null) { - for (int i = 0; i < size; i++) { - Object key = fury.readRef(buffer, keySerializer); - Object value = fury.readRef(buffer, valueSerializer); - map.put(key, value); - } - } else if (keySerializer != null) { - for (int i = 0; i < size; i++) { - Object key = fury.readRef(buffer, keySerializer); - map.put(key, fury.readRef(buffer, keyClassInfoReadCache)); - } - } else if (valueSerializer != null) { - for (int i = 0; i < size; i++) { - Object key = fury.readRef(buffer); - Object value = fury.readRef(buffer, valueSerializer); - map.put(key, value); - } - } else { - genericJavaRead(fury, buffer, map, size); - } + private void javaChunkReadWithValueSerializer( + MemoryBuffer buffer, Map map, int size, Serializer valueSerializer) { + MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); + while (size > 0) { + byte chunkSize = buffer.readByte(); + byte header = buffer.readByte(); + Preconditions.checkArgument( + chunkSize >= 0, + "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); + for (byte i = 0; i < chunkSize; i++) { + Object key; + Object value; + key = + mapChunkWriter.readKey( + header, buffer, fury.getClassResolver(), fury.trackingRef(), keyClassInfoReadCache); + value = mapChunkWriter.readFinalValue(buffer, header, valueSerializer); + map.put(key, value); + size--; + } } + } - private void genericJavaRead(Fury fury, MemoryBuffer buffer, Map map, int size) { - Generics generics = fury.getGenerics(); - GenericType genericType = generics.nextGenericType(); - if (genericType == null) { - generalJavaRead(fury, buffer, map, size); - } else { - GenericType keyGenericType = genericType.getTypeParameter0(); - GenericType valueGenericType = genericType.getTypeParameter1(); - if (genericType.getTypeParametersCount() < 2) { - Tuple2 kvGenericType = getKVGenericType(genericType); - if (keyGenericType == objType && valueGenericType == objType) { - generalJavaRead(fury, buffer, map, size); - return; - } - keyGenericType = kvGenericType.f0; - valueGenericType = kvGenericType.f1; - } - boolean keyGenericTypeFinal = keyGenericType.isMonomorphic(); - boolean valueGenericTypeFinal = valueGenericType.isMonomorphic(); - if (keyGenericTypeFinal && valueGenericTypeFinal) { - javaKVTypesFinalRead(fury, buffer, map, keyGenericType, valueGenericType, generics, size); - } else if (keyGenericTypeFinal) { - javaKeyTypeFinalRead(fury, buffer, map, keyGenericType, valueGenericType, generics, size); - } else if (valueGenericTypeFinal) { - javaValueTypeFinalRead(fury, buffer, map, keyGenericType, valueGenericType, generics, size); - } else { - javaKVTypesNonFinalRead( - fury, buffer, map, keyGenericType, valueGenericType, generics, size); - } - generics.popGenericType(); - } + private void javaChunkReadWithKVSerializers( + MemoryBuffer buffer, + Map map, + int size, + Serializer keySerializer, + Serializer valueSerializer) { + MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); + while (size > 0) { + byte chunkSize = buffer.readByte(); + byte header = buffer.readByte(); + Preconditions.checkArgument( + chunkSize >= 0, + "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); + for (byte i = 0; i < chunkSize; i++) { + Object key; + Object value; + key = mapChunkWriter.readFinalKey(buffer, header, keySerializer); + value = mapChunkWriter.readFinalValue(buffer, header, valueSerializer); + map.put(key, value); + size--; + } } + } - private void javaChunkReadWithKeySerializer(MemoryBuffer buffer, Map map, int size, Serializer keySerializer) { - MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); - while (size > 0) { - byte chunkSize = buffer.readByte(); - byte header = buffer.readByte(); - Preconditions.checkArgument(chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); - for (byte i = 0; i < chunkSize; i++) { - Object key; - Object value; - key = mapChunkWriter.readFinalKey(buffer, header, keySerializer); - value = mapChunkWriter.readValue(header, buffer, fury.getClassResolver(), fury.trackingRef(), valueClassInfoReadCache); - map.put(key, value); - size--; - } + private void genericJavaChunkRead(Fury fury, MemoryBuffer buffer, Map map, int size) { + Generics generics = fury.getGenerics(); + GenericType genericType = generics.nextGenericType(); + if (genericType == null) { + generalJavaChunkRead(fury, buffer, map, size); + } else { + GenericType keyGenericType = genericType.getTypeParameter0(); + GenericType valueGenericType = genericType.getTypeParameter1(); + if (genericType.getTypeParametersCount() < 2) { + Tuple2 kvGenericType = getKVGenericType(genericType); + if (keyGenericType == objType && valueGenericType == objType) { + generalJavaChunkRead(fury, buffer, map, size); + return; } + keyGenericType = kvGenericType.f0; + valueGenericType = kvGenericType.f1; + } + boolean keyGenericTypeFinal = keyGenericType.isMonomorphic(); + boolean valueGenericTypeFinal = valueGenericType.isMonomorphic(); + if (keyGenericTypeFinal && valueGenericTypeFinal) { + javaKVTypesFinalChunkRead( + fury, buffer, map, keyGenericType, valueGenericType, generics, size); + } else if (keyGenericTypeFinal) { + javaKeyTypeFinalChunkRead( + fury, buffer, map, keyGenericType, valueGenericType, generics, size); + } else if (valueGenericTypeFinal) { + javaValueTypeFinalChunkRead( + fury, buffer, map, keyGenericType, valueGenericType, generics, size); + } else { + javaKVTypesNonFinalChunkRead( + fury, buffer, map, keyGenericType, valueGenericType, generics, size); + } + generics.popGenericType(); } + } - private void javaChunkReadWithValueSerializer(MemoryBuffer buffer, Map map, int size, Serializer valueSerializer) { - MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); - while (size > 0) { - byte chunkSize = buffer.readByte(); - byte header = buffer.readByte(); - Preconditions.checkArgument(chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); - for (byte i = 0; i < chunkSize; i++) { - Object key; - Object value; - key = mapChunkWriter.readKey(header, buffer, fury.getClassResolver(), fury.trackingRef(), keyClassInfoReadCache); - value = mapChunkWriter.readFinalValue(buffer, header, valueSerializer); - map.put(key, value); - size--; - } - } + private void javaKVTypesFinalChunkRead( + Fury fury, + MemoryBuffer buffer, + Map map, + GenericType keyGenericType, + GenericType valueGenericType, + Generics generics, + int size) { + Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); + Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); + MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); + while (size > 0) { + byte chunkSize = buffer.readByte(); + byte header = buffer.readByte(); + mapChunkWriter.setKeySerializer(null); + mapChunkWriter.setValueSerializer(null); + Preconditions.checkArgument( + chunkSize >= 0, + "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); + for (byte i = 0; i < chunkSize; i++) { + Object key; + Object value; + generics.pushGenericType(keyGenericType); + key = mapChunkWriter.readFinalKey(buffer, header, keySerializer); + generics.popGenericType(); + generics.pushGenericType(valueGenericType); + value = mapChunkWriter.readFinalValue(buffer, header, valueSerializer); + generics.popGenericType(); + map.put(key, value); + size--; + } } + } - private void javaChunkReadWithKVSerializers(MemoryBuffer buffer, Map map, int size, Serializer keySerializer, Serializer valueSerializer) { - MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); - while (size > 0) { - byte chunkSize = buffer.readByte(); - byte header = buffer.readByte(); - Preconditions.checkArgument(chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); - for (byte i = 0; i < chunkSize; i++) { - Object key; - Object value; - key = mapChunkWriter.readFinalKey(buffer, header, keySerializer); - value = mapChunkWriter.readFinalValue(buffer, header, valueSerializer); - map.put(key, value); - size--; - } - } + private void javaKVTypesFinalRead( + Fury fury, + MemoryBuffer buffer, + Map map, + GenericType keyGenericType, + GenericType valueGenericType, + Generics generics, + int size) { + Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); + Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); + for (int i = 0; i < size; i++) { + generics.pushGenericType(keyGenericType); + Object key = fury.readRef(buffer, keySerializer); + generics.popGenericType(); + generics.pushGenericType(valueGenericType); + Object value = fury.readRef(buffer, valueSerializer); + generics.popGenericType(); + map.put(key, value); } + } - private void genericJavaChunkRead(Fury fury, MemoryBuffer buffer, Map map, int size) { - Generics generics = fury.getGenerics(); - GenericType genericType = generics.nextGenericType(); - if (genericType == null) { - generalJavaChunkRead(fury, buffer, map, size); - } else { - GenericType keyGenericType = genericType.getTypeParameter0(); - GenericType valueGenericType = genericType.getTypeParameter1(); - if (genericType.getTypeParametersCount() < 2) { - Tuple2 kvGenericType = getKVGenericType(genericType); - if (keyGenericType == objType && valueGenericType == objType) { - generalJavaChunkRead(fury, buffer, map, size); - return; - } - keyGenericType = kvGenericType.f0; - valueGenericType = kvGenericType.f1; - } - boolean keyGenericTypeFinal = keyGenericType.isMonomorphic(); - boolean valueGenericTypeFinal = valueGenericType.isMonomorphic(); - if (keyGenericTypeFinal && valueGenericTypeFinal) { - javaKVTypesFinalChunkRead(fury, buffer, map, keyGenericType, valueGenericType, generics, size); - } else if (keyGenericTypeFinal) { - javaKeyTypeFinalChunkRead(fury, buffer, map, keyGenericType, valueGenericType, generics, size); - } else if (valueGenericTypeFinal) { - javaValueTypeFinalChunkRead(fury, buffer, map, keyGenericType, valueGenericType, generics, size); - } else { - javaKVTypesNonFinalChunkRead( - fury, buffer, map, keyGenericType, valueGenericType, generics, size); - } - generics.popGenericType(); - } + private void javaKeyTypeFinalChunkRead( + Fury fury, + MemoryBuffer buffer, + Map map, + GenericType keyGenericType, + GenericType valueGenericType, + Generics generics, + int size) { + final MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); + ClassResolver classResolver = fury.getClassResolver(); + boolean trackingValueRef = classResolver.needToWriteRef(valueGenericType.getCls()); + Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); + while (size > 0) { + byte chunkSize = buffer.readByte(); + Preconditions.checkArgument( + chunkSize >= 0, + "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); + byte header = buffer.readByte(); + mapChunkWriter.setKeySerializer(null); + mapChunkWriter.setValueSerializer(null); + while (chunkSize > 0) { + generics.pushGenericType(keyGenericType); + Object key = mapChunkWriter.readFinalKey(buffer, header, keySerializer); + generics.pushGenericType(valueGenericType); + Object value = + mapChunkWriter.readValue( + header, buffer, classResolver, trackingValueRef, valueClassInfoReadCache); + generics.pushGenericType(valueGenericType); + generics.popGenericType(); + chunkSize--; + size--; + map.put(key, value); + } } + } - private void javaKVTypesFinalChunkRead(Fury fury, - MemoryBuffer buffer, - Map map, - GenericType keyGenericType, - GenericType valueGenericType, - Generics generics, - int size) { - Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); - Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); - MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); - while (size > 0) { - byte chunkSize = buffer.readByte(); - byte header = buffer.readByte(); - mapChunkWriter.setKeySerializer(null); - mapChunkWriter.setValueSerializer(null); - Preconditions.checkArgument(chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); - for (byte i = 0; i < chunkSize; i++) { - Object key; - Object value; - generics.pushGenericType(keyGenericType); - key = mapChunkWriter.readFinalKey(buffer, header, keySerializer); - generics.popGenericType(); - generics.pushGenericType(valueGenericType); - value = mapChunkWriter.readFinalValue(buffer, header, valueSerializer); - generics.popGenericType(); - map.put(key, value); - size--; - } - } + private void javaKeyTypeFinalRead( + Fury fury, + MemoryBuffer buffer, + Map map, + GenericType keyGenericType, + GenericType valueGenericType, + Generics generics, + int size) { + RefResolver refResolver = fury.getRefResolver(); + boolean trackingValueRef = fury.getClassResolver().needToWriteRef(valueGenericType.getCls()); + Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); + for (int i = 0; i < size; i++) { + generics.pushGenericType(keyGenericType); + Object key = fury.readRef(buffer, keySerializer); + generics.popGenericType(); + generics.pushGenericType(valueGenericType); + Object value = + readJavaRefOptimized( + fury, refResolver, trackingValueRef, buffer, valueClassInfoWriteCache); + generics.popGenericType(); + map.put(key, value); + } + } + private void javaValueTypeFinalChunkRead( + Fury fury, + MemoryBuffer buffer, + Map map, + GenericType keyGenericType, + GenericType valueGenericType, + Generics generics, + int size) { + MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); + boolean trackingKeyRef = fury.getClassResolver().needToWriteRef(keyGenericType.getCls()); + Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); + ClassResolver classResolver = fury.getClassResolver(); + while (size > 0) { + byte chunkSize = buffer.readByte(); + Preconditions.checkArgument( + chunkSize >= 0, + "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); + byte header = buffer.readByte(); + mapChunkWriter.setKeySerializer(null); + mapChunkWriter.setValueSerializer(null); + while (chunkSize > 0) { + generics.pushGenericType(keyGenericType); + Object key = + mapChunkWriter.readKey( + header, buffer, classResolver, trackingKeyRef, keyClassInfoReadCache); + generics.pushGenericType(valueGenericType); + Object value = mapChunkWriter.readFinalValue(buffer, header, valueSerializer); + generics.pushGenericType(valueGenericType); + generics.popGenericType(); + chunkSize--; + size--; + map.put(key, value); + } } + } - private void javaKVTypesFinalRead( - Fury fury, - MemoryBuffer buffer, - Map map, - GenericType keyGenericType, - GenericType valueGenericType, - Generics generics, - int size) { - Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); - Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); - for (int i = 0; i < size; i++) { - generics.pushGenericType(keyGenericType); - Object key = fury.readRef(buffer, keySerializer); - generics.popGenericType(); - generics.pushGenericType(valueGenericType); - Object value = fury.readRef(buffer, valueSerializer); - generics.popGenericType(); - map.put(key, value); - } + private void javaValueTypeFinalRead( + Fury fury, + MemoryBuffer buffer, + Map map, + GenericType keyGenericType, + GenericType valueGenericType, + Generics generics, + int size) { + boolean trackingKeyRef = fury.getClassResolver().needToWriteRef(keyGenericType.getCls()); + Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); + RefResolver refResolver = fury.getRefResolver(); + for (int i = 0; i < size; i++) { + generics.pushGenericType(keyGenericType); + Object key = + readJavaRefOptimized(fury, refResolver, trackingKeyRef, buffer, keyClassInfoWriteCache); + generics.popGenericType(); + generics.pushGenericType(valueGenericType); + Object value = fury.readRef(buffer, valueSerializer); + generics.popGenericType(); + map.put(key, value); } + } - private void javaKeyTypeFinalChunkRead(Fury fury, - MemoryBuffer buffer, - Map map, - GenericType keyGenericType, - GenericType valueGenericType, - Generics generics, - int size) { - final MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); - ClassResolver classResolver = fury.getClassResolver(); - boolean trackingValueRef = classResolver.needToWriteRef(valueGenericType.getCls()); - Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); + private void javaKVTypesNonFinalChunkRead( + Fury fury, + MemoryBuffer buffer, + Map map, + GenericType keyGenericType, + GenericType valueGenericType, + Generics generics, + int size) { + final MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); + ClassResolver classResolver = fury.getClassResolver(); + RefResolver refResolver = fury.getRefResolver(); + boolean trackingKeyRef = classResolver.needToWriteRef(keyGenericType.getCls()); + boolean trackingValueRef = classResolver.needToWriteRef(valueGenericType.getCls()); + while (size > 0) { + byte chunkSize = buffer.readByte(); + Preconditions.checkArgument( + chunkSize >= 0, + "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); + if (chunkSize == 0) { while (size > 0) { - byte chunkSize = buffer.readByte(); - Preconditions.checkArgument(chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); - byte header = buffer.readByte(); - mapChunkWriter.setKeySerializer(null); - mapChunkWriter.setValueSerializer(null); - while (chunkSize > 0) { - generics.pushGenericType(keyGenericType); - Object key = mapChunkWriter.readFinalKey(buffer, header, keySerializer); - generics.pushGenericType(valueGenericType); - Object value = mapChunkWriter.readValue(header, buffer, classResolver, trackingValueRef, valueClassInfoReadCache); - generics.pushGenericType(valueGenericType); - generics.popGenericType(); - chunkSize--; - size--; - map.put(key, value); - } + generics.pushGenericType(keyGenericType); + Object key = + readJavaRefOptimized( + fury, refResolver, trackingKeyRef, buffer, keyClassInfoReadCache); + generics.popGenericType(); + generics.pushGenericType(valueGenericType); + Object value = + readJavaRefOptimized( + fury, refResolver, trackingValueRef, buffer, valueClassInfoReadCache); + generics.popGenericType(); + map.put(key, value); + size--; } + } else { + byte header = buffer.readByte(); + mapChunkWriter.setKeySerializer(null); + mapChunkWriter.setValueSerializer(null); + while (chunkSize > 0) { + generics.pushGenericType(keyGenericType); + Object key = + mapChunkWriter.readKey( + header, buffer, classResolver, trackingKeyRef, keyClassInfoReadCache); + generics.popGenericType(); + generics.pushGenericType(valueGenericType); + Object value = + mapChunkWriter.readValue( + header, buffer, classResolver, trackingValueRef, valueClassInfoReadCache); + generics.popGenericType(); + chunkSize--; + size--; + map.put(key, value); + } + } } + } - private void javaKeyTypeFinalRead(Fury fury - , MemoryBuffer buffer - , Map map - , GenericType keyGenericType - , GenericType valueGenericType - , Generics generics - , int size) { - RefResolver refResolver = fury.getRefResolver(); - boolean trackingValueRef = fury.getClassResolver().needToWriteRef(valueGenericType.getCls()); - Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); - for (int i = 0; i < size; i++) { - generics.pushGenericType(keyGenericType); - Object key = fury.readRef(buffer, keySerializer); - generics.popGenericType(); - generics.pushGenericType(valueGenericType); - Object value = - readJavaRefOptimized( - fury, refResolver, trackingValueRef, buffer, valueClassInfoWriteCache); - generics.popGenericType(); - map.put(key, value); - } + private void javaKVTypesNonFinalRead( + Fury fury, + MemoryBuffer buffer, + Map map, + GenericType keyGenericType, + GenericType valueGenericType, + Generics generics, + int size) { + ClassResolver classResolver = fury.getClassResolver(); + RefResolver refResolver = fury.getRefResolver(); + boolean trackingKeyRef = classResolver.needToWriteRef(keyGenericType.getCls()); + boolean trackingValueRef = classResolver.needToWriteRef(valueGenericType.getCls()); + for (int i = 0; i < size; i++) { + generics.pushGenericType(keyGenericType); + Object key = + readJavaRefOptimized(fury, refResolver, trackingKeyRef, buffer, keyClassInfoWriteCache); + generics.popGenericType(); + generics.pushGenericType(valueGenericType); + Object value = + readJavaRefOptimized( + fury, refResolver, trackingValueRef, buffer, valueClassInfoWriteCache); + generics.popGenericType(); + map.put(key, value); } + } - private void javaValueTypeFinalChunkRead(Fury fury - , MemoryBuffer buffer - , Map map - , GenericType keyGenericType - , GenericType valueGenericType - , Generics generics - , int size) { - MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); - boolean trackingKeyRef = fury.getClassResolver().needToWriteRef(keyGenericType.getCls()); - Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); - ClassResolver classResolver = fury.getClassResolver(); + private void generalJavaChunkRead(Fury fury, MemoryBuffer buffer, Map map, int size) { + final MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); + ClassResolver classResolver = fury.getClassResolver(); + boolean trackingKeyRef = fury.trackingRef(); + while (size > 0) { + byte chunkSize = buffer.readByte(); + Preconditions.checkArgument( + chunkSize >= 0, + "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); + if (chunkSize == 0) { while (size > 0) { - byte chunkSize = buffer.readByte(); - Preconditions.checkArgument(chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); - byte header = buffer.readByte(); - mapChunkWriter.setKeySerializer(null); - mapChunkWriter.setValueSerializer(null); - while (chunkSize > 0) { - generics.pushGenericType(keyGenericType); - Object key = mapChunkWriter.readKey(header, buffer, classResolver, trackingKeyRef, keyClassInfoReadCache); - generics.pushGenericType(valueGenericType); - Object value = mapChunkWriter.readFinalValue(buffer, header, valueSerializer); - generics.pushGenericType(valueGenericType); - generics.popGenericType(); - chunkSize--; - size--; - map.put(key, value); - } + Object key = fury.readRef(buffer, keyClassInfoReadCache); + Object value = fury.readRef(buffer, keyClassInfoReadCache); + map.put(key, value); + size--; } - } - - private void javaValueTypeFinalRead( - Fury fury, - MemoryBuffer buffer, - Map map, - GenericType keyGenericType, - GenericType valueGenericType, - Generics generics, - int size) { - boolean trackingKeyRef = fury.getClassResolver().needToWriteRef(keyGenericType.getCls()); - Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); - RefResolver refResolver = fury.getRefResolver(); - for (int i = 0; i < size; i++) { - generics.pushGenericType(keyGenericType); - Object key = - readJavaRefOptimized(fury, refResolver, trackingKeyRef, buffer, keyClassInfoWriteCache); - generics.popGenericType(); - generics.pushGenericType(valueGenericType); - Object value = fury.readRef(buffer, valueSerializer); - generics.popGenericType(); - map.put(key, value); + } else { + byte header = buffer.readByte(); + mapChunkWriter.setKeySerializer(null); + mapChunkWriter.setValueSerializer(null); + while (chunkSize > 0) { + Object key = + mapChunkWriter.readKey( + header, buffer, classResolver, trackingKeyRef, keyClassInfoReadCache); + Object value = + mapChunkWriter.readValue( + header, buffer, classResolver, trackingKeyRef, valueClassInfoReadCache); + chunkSize--; + size--; + map.put(key, value); } + } } + } - private void javaKVTypesNonFinalChunkRead(Fury fury, - MemoryBuffer buffer, - Map map, - GenericType keyGenericType, - GenericType valueGenericType, - Generics generics, - int size) { - final MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); - ClassResolver classResolver = fury.getClassResolver(); - RefResolver refResolver = fury.getRefResolver(); - boolean trackingKeyRef = classResolver.needToWriteRef(keyGenericType.getCls()); - boolean trackingValueRef = classResolver.needToWriteRef(valueGenericType.getCls()); - while (size > 0) { - byte chunkSize = buffer.readByte(); - Preconditions.checkArgument(chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); - if (chunkSize == 0) { - while (size > 0) { - generics.pushGenericType(keyGenericType); - Object key = readJavaRefOptimized(fury, refResolver, trackingKeyRef, buffer, keyClassInfoReadCache); - generics.popGenericType(); - generics.pushGenericType(valueGenericType); - Object value = readJavaRefOptimized(fury, refResolver, trackingValueRef, buffer, valueClassInfoReadCache); - generics.popGenericType(); - map.put(key, value); - size--; - } - } else { - byte header = buffer.readByte(); - mapChunkWriter.setKeySerializer(null); - mapChunkWriter.setValueSerializer(null); - while (chunkSize > 0) { - generics.pushGenericType(keyGenericType); - Object key = mapChunkWriter.readKey(header, buffer, classResolver, trackingKeyRef, keyClassInfoReadCache); - generics.popGenericType(); - generics.pushGenericType(valueGenericType); - Object value = mapChunkWriter.readValue(header, buffer, classResolver, trackingValueRef, valueClassInfoReadCache); - generics.popGenericType(); - chunkSize--; - size--; - map.put(key, value); - } - } - } + private void generalJavaRead(Fury fury, MemoryBuffer buffer, Map map, int size) { + for (int i = 0; i < size; i++) { + Object key = fury.readRef(buffer, keyClassInfoReadCache); + Object value = fury.readRef(buffer, valueClassInfoReadCache); + map.put(key, value); } + } - private void javaKVTypesNonFinalRead( - Fury fury, - MemoryBuffer buffer, - Map map, - GenericType keyGenericType, - GenericType valueGenericType, - Generics generics, - int size) { - ClassResolver classResolver = fury.getClassResolver(); - RefResolver refResolver = fury.getRefResolver(); - boolean trackingKeyRef = classResolver.needToWriteRef(keyGenericType.getCls()); - boolean trackingValueRef = classResolver.needToWriteRef(valueGenericType.getCls()); + @SuppressWarnings("unchecked") + public static void xreadElements(Fury fury, MemoryBuffer buffer, Map map, int size) { + Generics generics = fury.getGenerics(); + GenericType genericType = generics.nextGenericType(); + if (genericType == null || genericType.getTypeParametersCount() != 2) { + for (int i = 0; i < size; i++) { + Object key = fury.xreadRef(buffer); + Object value = fury.xreadRef(buffer); + map.put(key, value); + } + } else { + // TODO(chaokunyang) use codegen to remove all branches. + GenericType keyGenericType = genericType.getTypeParameter0(); + GenericType valueGenericType = genericType.getTypeParameter1(); + Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); + Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); + if (!keyGenericType.hasGenericParameters() && !valueGenericType.hasGenericParameters()) { for (int i = 0; i < size; i++) { - generics.pushGenericType(keyGenericType); - Object key = - readJavaRefOptimized(fury, refResolver, trackingKeyRef, buffer, keyClassInfoWriteCache); - generics.popGenericType(); - generics.pushGenericType(valueGenericType); - Object value = - readJavaRefOptimized( - fury, refResolver, trackingValueRef, buffer, valueClassInfoWriteCache); - generics.popGenericType(); - map.put(key, value); + Object key = fury.xreadRefByNullableSerializer(buffer, keySerializer); + Object value = fury.xreadRefByNullableSerializer(buffer, valueSerializer); + map.put(key, value); } - } - - - private void generalJavaChunkRead(Fury fury, MemoryBuffer buffer, Map map, int size) { - final MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); - ClassResolver classResolver = fury.getClassResolver(); - boolean trackingKeyRef = fury.trackingRef(); - while (size > 0) { - byte chunkSize = buffer.readByte(); - Preconditions.checkArgument(chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); - if (chunkSize == 0) { - while (size > 0) { - Object key = fury.readRef(buffer, keyClassInfoReadCache); - Object value = fury.readRef(buffer, keyClassInfoReadCache); - map.put(key, value); - size--; - } - } else { - byte header = buffer.readByte(); - mapChunkWriter.setKeySerializer(null); - mapChunkWriter.setValueSerializer(null); - while (chunkSize > 0) { - Object key = mapChunkWriter.readKey(header, buffer, classResolver, trackingKeyRef, keyClassInfoReadCache); - Object value = mapChunkWriter.readValue(header, buffer, classResolver, trackingKeyRef, valueClassInfoReadCache); - chunkSize--; - size--; - map.put(key, value); - } - } + } else if (valueGenericType.hasGenericParameters()) { + for (int i = 0; i < size; i++) { + Object key = fury.xreadRefByNullableSerializer(buffer, keySerializer); + generics.pushGenericType(valueGenericType); + Object value = fury.xreadRefByNullableSerializer(buffer, valueSerializer); + generics.popGenericType(); + map.put(key, value); } - } - - private void generalJavaRead(Fury fury, MemoryBuffer buffer, Map map, int size) { + } else if (keyGenericType.hasGenericParameters()) { for (int i = 0; i < size; i++) { - Object key = fury.readRef(buffer, keyClassInfoReadCache); - Object value = fury.readRef(buffer, valueClassInfoReadCache); - map.put(key, value); + generics.pushGenericType(keyGenericType); + Object key = fury.xreadRefByNullableSerializer(buffer, keySerializer); + generics.popGenericType(); + Object value = fury.xreadRefByNullableSerializer(buffer, valueSerializer); + map.put(key, value); } - } - - @SuppressWarnings("unchecked") - public static void xreadElements(Fury fury, MemoryBuffer buffer, Map map, int size) { - Generics generics = fury.getGenerics(); - GenericType genericType = generics.nextGenericType(); - if (genericType == null || genericType.getTypeParametersCount() != 2) { - for (int i = 0; i < size; i++) { - Object key = fury.xreadRef(buffer); - Object value = fury.xreadRef(buffer); - map.put(key, value); - } - } else { - // TODO(chaokunyang) use codegen to remove all branches. - GenericType keyGenericType = genericType.getTypeParameter0(); - GenericType valueGenericType = genericType.getTypeParameter1(); - Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); - Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); - if (!keyGenericType.hasGenericParameters() && !valueGenericType.hasGenericParameters()) { - for (int i = 0; i < size; i++) { - Object key = fury.xreadRefByNullableSerializer(buffer, keySerializer); - Object value = fury.xreadRefByNullableSerializer(buffer, valueSerializer); - map.put(key, value); - } - } else if (valueGenericType.hasGenericParameters()) { - for (int i = 0; i < size; i++) { - Object key = fury.xreadRefByNullableSerializer(buffer, keySerializer); - generics.pushGenericType(valueGenericType); - Object value = fury.xreadRefByNullableSerializer(buffer, valueSerializer); - generics.popGenericType(); - map.put(key, value); - } - } else if (keyGenericType.hasGenericParameters()) { - for (int i = 0; i < size; i++) { - generics.pushGenericType(keyGenericType); - Object key = fury.xreadRefByNullableSerializer(buffer, keySerializer); - generics.popGenericType(); - Object value = fury.xreadRefByNullableSerializer(buffer, valueSerializer); - map.put(key, value); - } - } else { - for (int i = 0; i < size; i++) { - // FIXME(chaokunyang) nested generics may be get by mistake. - generics.pushGenericType(keyGenericType); - Object key = fury.xreadRefByNullableSerializer(buffer, keySerializer); - generics.pushGenericType(valueGenericType); - Object value = fury.xreadRefByNullableSerializer(buffer, valueSerializer); - map.put(key, value); - } - } - generics.popGenericType(); + } else { + for (int i = 0; i < size; i++) { + // FIXME(chaokunyang) nested generics may be get by mistake. + generics.pushGenericType(keyGenericType); + Object key = fury.xreadRefByNullableSerializer(buffer, keySerializer); + generics.pushGenericType(valueGenericType); + Object value = fury.xreadRefByNullableSerializer(buffer, valueSerializer); + map.put(key, value); } + } + generics.popGenericType(); } + } - /** - * Hook for java serialization codegen, read/write key/value by entrySet. - * - *

For key/value type which is final, using codegen may get a big performance gain - * - * @return true if read/write key/value support calling entrySet method - */ - public final boolean supportCodegenHook() { - return supportCodegenHook; - } + /** + * Hook for java serialization codegen, read/write key/value by entrySet. + * + *

For key/value type which is final, using codegen may get a big performance gain + * + * @return true if read/write key/value support calling entrySet method + */ + public final boolean supportCodegenHook() { + return supportCodegenHook; + } - /** - * Write data except size and elements. - * - *

    - * In codegen, follows is call order: - *
  1. write map class if not final - *
  2. write map size - *
  3. onCollectionWrite - *
  4. write keys/values - *
- */ - public abstract Map onMapWrite(MemoryBuffer buffer, T value); + /** + * Write data except size and elements. + * + *
    + * In codegen, follows is call order: + *
  1. write map class if not final + *
  2. write map size + *
  3. onCollectionWrite + *
  4. write keys/values + *
+ */ + public abstract Map onMapWrite(MemoryBuffer buffer, T value); - /** - * Check null first to avoid ref tracking for some types with ref tracking disabled. - */ - private void writeJavaRefOptimized( - Fury fury, - ClassResolver classResolver, - RefResolver refResolver, - MemoryBuffer buffer, - Object obj, - ClassInfoHolder classInfoHolder) { - if (!refResolver.writeNullFlag(buffer, obj)) { - fury.writeRef(buffer, obj, classResolver.getClassInfo(obj.getClass(), classInfoHolder)); - } + /** Check null first to avoid ref tracking for some types with ref tracking disabled. */ + private void writeJavaRefOptimized( + Fury fury, + ClassResolver classResolver, + RefResolver refResolver, + MemoryBuffer buffer, + Object obj, + ClassInfoHolder classInfoHolder) { + if (!refResolver.writeNullFlag(buffer, obj)) { + fury.writeRef(buffer, obj, classResolver.getClassInfo(obj.getClass(), classInfoHolder)); } + } - private void writeJavaRefOptimized( - Fury fury, - ClassResolver classResolver, - RefResolver refResolver, - boolean trackingRef, - MemoryBuffer buffer, - Object obj, - ClassInfoHolder classInfoHolder) { - if (trackingRef) { - if (!refResolver.writeNullFlag(buffer, obj)) { - fury.writeRef(buffer, obj, classResolver.getClassInfo(obj.getClass(), classInfoHolder)); - } - } else { - if (obj == null) { - buffer.writeByte(Fury.NULL_FLAG); - } else { - buffer.writeByte(Fury.NOT_NULL_VALUE_FLAG); - fury.writeNonRef(buffer, obj, classResolver.getClassInfo(obj.getClass(), classInfoHolder)); - } - } + private void writeJavaRefOptimized( + Fury fury, + ClassResolver classResolver, + RefResolver refResolver, + boolean trackingRef, + MemoryBuffer buffer, + Object obj, + ClassInfoHolder classInfoHolder) { + if (trackingRef) { + if (!refResolver.writeNullFlag(buffer, obj)) { + fury.writeRef(buffer, obj, classResolver.getClassInfo(obj.getClass(), classInfoHolder)); + } + } else { + if (obj == null) { + buffer.writeByte(Fury.NULL_FLAG); + } else { + buffer.writeByte(Fury.NOT_NULL_VALUE_FLAG); + fury.writeNonRef(buffer, obj, classResolver.getClassInfo(obj.getClass(), classInfoHolder)); + } } + } - @Override - public abstract T read(MemoryBuffer buffer); + @Override + public abstract T read(MemoryBuffer buffer); - /** - * Read data except size and elements, return empty map to be filled. - * - *
    - * In codegen, follows is call order: - *
  1. read map class if not final - *
  2. newMap: read and set map size, read map header and create map. - *
  3. read keys/values - *
- * - *

Map must have default constructor to be invoked by fury, otherwise created object can't be - * used to adding elements. For example: - * - * 

{@code new ArrayList {add(1);}}
- * - *

without default constructor, created list will have elementData as null, adding elements - * will raise NPE. - */ - public Map newMap(MemoryBuffer buffer) { - numElements = buffer.readVarUint32Small7(); - if (constructor == null) { - constructor = ReflectionUtils.getCtrHandle(type, true); - } - try { - Map instance = (Map) constructor.invoke(); - fury.getRefResolver().reference(instance); - return instance; - } catch (Throwable e) { - throw new IllegalArgumentException( - "Please provide public no arguments constructor for class " + type, e); - } + /** + * Read data except size and elements, return empty map to be filled. + * + *

    + * In codegen, follows is call order: + *
  1. read map class if not final + *
  2. newMap: read and set map size, read map header and create map. + *
  3. read keys/values + *
+ * + *

Map must have default constructor to be invoked by fury, otherwise created object can't be + * used to adding elements. For example: + * + * 

{@code new ArrayList {add(1);}}
+ * + *

without default constructor, created list will have elementData as null, adding elements + * will raise NPE. + */ + public Map newMap(MemoryBuffer buffer) { + numElements = buffer.readVarUint32Small7(); + if (constructor == null) { + constructor = ReflectionUtils.getCtrHandle(type, true); } - - /** - * Get and reset numElements of deserializing collection. Should be called after {@link #newMap}. - * Nested read may overwrite this element, reset is necessary to avoid use wrong value by mistake. - */ - public int getAndClearNumElements() { - int size = numElements; - numElements = -1; // nested read may overwrite this element. - return size; + try { + Map instance = (Map) constructor.invoke(); + fury.getRefResolver().reference(instance); + return instance; + } catch (Throwable e) { + throw new IllegalArgumentException( + "Please provide public no arguments constructor for class " + type, e); } + } - public void setNumElements(int numElements) { - this.numElements = numElements; - } + /** + * Get and reset numElements of deserializing collection. Should be called after {@link #newMap}. + * Nested read may overwrite this element, reset is necessary to avoid use wrong value by mistake. + */ + public int getAndClearNumElements() { + int size = numElements; + numElements = -1; // nested read may overwrite this element. + return size; + } - public abstract T onMapRead(Map map); + public void setNumElements(int numElements) { + this.numElements = numElements; + } - private Object readJavaRefOptimized( - Fury fury, - RefResolver refResolver, - boolean trackingRef, - MemoryBuffer buffer, - ClassInfoHolder classInfoHolder) { - if (trackingRef) { - int nextReadRefId = refResolver.tryPreserveRefId(buffer); - if (nextReadRefId >= Fury.NOT_NULL_VALUE_FLAG) { - Object obj = fury.readNonRef(buffer, classInfoHolder); - refResolver.setReadObject(nextReadRefId, obj); - return obj; - } else { - return refResolver.getReadObject(); - } - } else { - byte headFlag = buffer.readByte(); - if (headFlag == Fury.NULL_FLAG) { - return null; - } else { - return fury.readNonRef(buffer, classInfoHolder); - } - } + public abstract T onMapRead(Map map); + + private Object readJavaRefOptimized( + Fury fury, + RefResolver refResolver, + boolean trackingRef, + MemoryBuffer buffer, + ClassInfoHolder classInfoHolder) { + if (trackingRef) { + int nextReadRefId = refResolver.tryPreserveRefId(buffer); + if (nextReadRefId >= Fury.NOT_NULL_VALUE_FLAG) { + Object obj = fury.readNonRef(buffer, classInfoHolder); + refResolver.setReadObject(nextReadRefId, obj); + return obj; + } else { + return refResolver.getReadObject(); + } + } else { + byte headFlag = buffer.readByte(); + if (headFlag == Fury.NULL_FLAG) { + return null; + } else { + return fury.readNonRef(buffer, classInfoHolder); + } } + } } diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapChunkWriter.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapChunkWriter.java index 7fa41d56c0..806189d536 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapChunkWriter.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapChunkWriter.java @@ -9,506 +9,541 @@ import org.apache.fury.serializer.Serializer; import org.apache.fury.util.Preconditions; -import java.util.Map; - public class MapChunkWriter { - private static final int MAX_CHUNK_SIZE = 127; - - public MapChunkWriter(Fury fury) { - this.fury = fury; + private static final int MAX_CHUNK_SIZE = 127; + + public MapChunkWriter(Fury fury) { + this.fury = fury; + } + + private int header = 0; + private int startOffset; + private int chunkSize; + private Class keyClass = null; + private Class valueClass = null; + private final Fury fury; + private boolean writeKeyClassInfo = false; + private boolean writeValueClassInfo = false; + private boolean keyIsNotSameType = false; + private boolean valueIsNotSameType = false; + private boolean prevKeyIsNull = false; + private Serializer keySerializer; + private Serializer valueSerializer; + + /** mark chunk write finish */ + private boolean markChunkWriteFinish = false; + + /** + * preserve two byte for header and chunk size and record the write index so that we can write key + * value at first, write header and chunk size when the chunk is finish at correct position + */ + private boolean hasPreservedByte = false; + + private void preserveByteForHeaderAndChunkSize(MemoryBuffer memoryBuffer) { + if (hasPreservedByte) { + return; } - - private int header = 0; - private int startOffset; - private int chunkSize; - private Class keyClass = null; - private Class valueClass = null; - private final Fury fury; - private boolean writeKeyClassInfo = false; - private boolean writeValueClassInfo = false; - private boolean keyIsNotSameType = false; - private boolean valueIsNotSameType = false; - private boolean prevKeyIsNull = false; - private Serializer keySerializer; - private Serializer valueSerializer; - - /** - * mark chunk write finish - */ - private boolean markChunkWriteFinish = false; - /** - * preserve two byte for header and chunk size and record the write index - * so that we can write key value at first, write header and chunk size when the chunk is finish at correct position - */ - private boolean hasPreservedByte = false; - - private void preserveByteForHeaderAndChunkSize(MemoryBuffer memoryBuffer) { - if (hasPreservedByte) { - return; + int writerIndex = memoryBuffer.writerIndex(); + // preserve two byte for header and chunk size + memoryBuffer.writerIndex(writerIndex + 2); + this.startOffset = writerIndex; + hasPreservedByte = true; + } + + public MapChunkWriter next(Object key, Object value, MemoryBuffer buffer) { + if (!markChunkWriteFinish) { + if (key == null) { + prevKeyIsNull = true; + if (chunkSize > 0) { + reset(buffer); } - int writerIndex = memoryBuffer.writerIndex(); - // preserve two byte for header and chunk size - memoryBuffer.writerIndex(writerIndex + 2); - this.startOffset = writerIndex; - hasPreservedByte = true; - } - - public MapChunkWriter next(Object key, Object value, MemoryBuffer buffer) { - if (!markChunkWriteFinish) { - if (key == null) { - prevKeyIsNull = true; - if (chunkSize > 0) { - reset(buffer); - } - } - if (prevKeyIsNull && key != null) { - reset(buffer); - } - if (value == null && chunkSize > 0 && !valueHasNull()) { - //if value has null before, no need to reset chunk - reset(buffer); - } - if (!keyIsNotSameType) { - this.keyIsNotSameType = judgeKeyIsNotSameType(key); - if (keyIsNotSameType) { - if (valueIsNotSameType) { - markChunkWriteFinish(buffer); - } else { - reset(buffer); - } - } - } - if (!valueIsNotSameType) { - this.valueIsNotSameType = judgeValueIsNotSameType(value); - if (valueIsNotSameType) { - if (keyIsNotSameType) { - markChunkWriteFinish(buffer); - } else { - reset(buffer); - } - } - } - if (chunkSize >= MAX_CHUNK_SIZE) { - reset(buffer); - } + } + if (prevKeyIsNull && key != null) { + reset(buffer); + } + if (value == null && chunkSize > 0 && !valueHasNull()) { + // if value has null before, no need to reset chunk + reset(buffer); + } + if (!keyIsNotSameType) { + this.keyIsNotSameType = judgeKeyIsNotSameType(key); + if (keyIsNotSameType) { + if (valueIsNotSameType) { + markChunkWriteFinish(buffer); + } else { + reset(buffer); + } } - return this; + } + if (!valueIsNotSameType) { + this.valueIsNotSameType = judgeValueIsNotSameType(value); + if (valueIsNotSameType) { + if (keyIsNotSameType) { + markChunkWriteFinish(buffer); + } else { + reset(buffer); + } + } + } + if (chunkSize >= MAX_CHUNK_SIZE) { + reset(buffer); + } } - - public void increaseChunkSize() { - chunkSize++; + return this; + } + + public void increaseChunkSize() { + chunkSize++; + } + + public void generalChunkWrite( + Object key, + Object value, + MemoryBuffer memoryBuffer, + ClassResolver classResolver, + RefResolver refResolver, + ClassInfoHolder keyClassInfoWriteCache, + ClassInfoHolder valueClassInfoWriteCache) { + final boolean trackingRef = fury.trackingRef(); + writeKey(key, memoryBuffer, classResolver, refResolver, trackingRef, keyClassInfoWriteCache); + writeValue( + value, memoryBuffer, classResolver, refResolver, trackingRef, valueClassInfoWriteCache); + increaseChunkSize(); + } + + public void writeFinalKey(Object key, MemoryBuffer buffer, Serializer keySerializer) { + preserveByteForHeaderAndChunkSize(buffer); + boolean trackingKeyRef = keySerializer.needToWriteRef(); + if (!trackingKeyRef) { + // map key has one null at most, use one chunk to write + if (key == null) { + header |= MapFlags.KEY_HAS_NULL; + buffer.writeByte(Fury.NULL_FLAG); + } else { + keySerializer.write(buffer, key); + } + } else { + updateFinalKeyHeader(key, true); + RefResolver refResolver = fury.getRefResolver(); + if (!refResolver.writeRefOrNull(buffer, key)) { + keySerializer.write(buffer, key); + } } - - public void generalChunkWrite(Object key, Object value, MemoryBuffer memoryBuffer, ClassResolver classResolver, RefResolver refResolver, ClassInfoHolder keyClassInfoWriteCache, ClassInfoHolder valueClassInfoWriteCache) { - final boolean trackingRef = fury.trackingRef(); - writeKey(key, memoryBuffer, classResolver, refResolver, trackingRef, keyClassInfoWriteCache); - writeValue(value, memoryBuffer, classResolver, refResolver, trackingRef, valueClassInfoWriteCache); - increaseChunkSize(); + } + + public Object readFinalKey(MemoryBuffer buffer, int header, Serializer keySerializer) { + this.header = header; + boolean trackingKeyRef = keySerializer.needToWriteRef(); + if (!trackingKeyRef) { + if (keyHasNull()) { + byte nullFlag = buffer.readByte(); + Preconditions.checkArgument(nullFlag == Fury.NULL_FLAG, "unexpected NULL_FLAG"); + return null; + } else { + return keySerializer.read(buffer); + } + } else { + return fury.readRef(buffer, keySerializer); } - - public void writeFinalKey(Object key, MemoryBuffer buffer, Serializer keySerializer) { - preserveByteForHeaderAndChunkSize(buffer); - boolean trackingKeyRef = keySerializer.needToWriteRef(); - if (!trackingKeyRef) { - // map key has one null at most, use one chunk to write - if (key == null) { - header |= MapFlags.KEY_HAS_NULL; - buffer.writeByte(Fury.NULL_FLAG); - } else { - keySerializer.write(buffer, key); - } + } + + public void writeFinalValue(Object value, MemoryBuffer buffer, Serializer valueSerializer) { + preserveByteForHeaderAndChunkSize(buffer); + boolean trackingValueRef = valueSerializer.needToWriteRef(); + if (!trackingValueRef) { + if (value == null) { + header |= MapFlags.VALUE_HAS_NULL; + buffer.writeByte(Fury.NULL_FLAG); + } else { + if (valueHasNull()) { + buffer.writeByte(Fury.NOT_NULL_VALUE_FLAG); + valueSerializer.write(buffer, value); } else { - updateFinalKeyHeader(key, true); - RefResolver refResolver = fury.getRefResolver(); - if (!refResolver.writeRefOrNull(buffer, key)) { - keySerializer.write(buffer, key); - } + valueSerializer.write(buffer, value); } + } + } else { + updateFinalValueHeader(value, true); + RefResolver refResolver = fury.getRefResolver(); + if (!refResolver.writeRefOrNull(buffer, value)) { + valueSerializer.write(buffer, value); + } } - - public Object readFinalKey(MemoryBuffer buffer, int header, Serializer keySerializer) { - this.header = header; - boolean trackingKeyRef = keySerializer.needToWriteRef(); - if (!trackingKeyRef) { - if (keyHasNull()) { - byte nullFlag = buffer.readByte(); - Preconditions.checkArgument(nullFlag == Fury.NULL_FLAG, "unexpected NULL_FLAG"); - return null; - } else { - return keySerializer.read(buffer); - } + } + + public Object readFinalValue(MemoryBuffer buffer, int header, Serializer valueSerializer) { + this.header = header; + boolean trackingValueRef = valueSerializer.needToWriteRef(); + if (!trackingValueRef) { + if (valueHasNull()) { + byte flag = buffer.readByte(); + if (flag == Fury.NOT_NULL_VALUE_FLAG) { + return valueSerializer.read(buffer); } else { - return fury.readRef(buffer, keySerializer); + return null; } + } else { + return valueSerializer.read(buffer); + } + } else { + return fury.readRef(buffer, valueSerializer); } + } - public void writeFinalValue(Object value, MemoryBuffer buffer, Serializer valueSerializer) { - preserveByteForHeaderAndChunkSize(buffer); - boolean trackingValueRef = valueSerializer.needToWriteRef(); - if (!trackingValueRef) { - if (value == null) { - header |= MapFlags.VALUE_HAS_NULL; - buffer.writeByte(Fury.NULL_FLAG); - } else { - if (valueHasNull()) { - buffer.writeByte(Fury.NOT_NULL_VALUE_FLAG); - valueSerializer.write(buffer, value); - } else { - valueSerializer.write(buffer, value); - } - } - } else { - updateFinalValueHeader(value, true); - RefResolver refResolver = fury.getRefResolver(); - if (!refResolver.writeRefOrNull(buffer, value)) { - valueSerializer.write(buffer, value); - } - } + private boolean judgeKeyIsNotSameType(Object key) { + if (key == null) { + return false; } - - public Object readFinalValue(MemoryBuffer buffer, int header, Serializer valueSerializer) { - this.header = header; - boolean trackingValueRef = valueSerializer.needToWriteRef(); - if (!trackingValueRef) { - if (valueHasNull()) { - byte flag = buffer.readByte(); - if (flag == Fury.NOT_NULL_VALUE_FLAG) { - return valueSerializer.read(buffer); - } else { - return null; - } - } else { - return valueSerializer.read(buffer); - } - } else { - return fury.readRef(buffer, valueSerializer); - } + if (keyClass == null) { + keyClass = key.getClass(); } + return keyClass != key.getClass(); + } - private boolean judgeKeyIsNotSameType(Object key) { - if (key == null) { - return false; - } - if (keyClass == null) { - keyClass = key.getClass(); - } - return keyClass != key.getClass(); + private boolean judgeValueIsNotSameType(Object value) { + if (value == null) { + return false; } - - private boolean judgeValueIsNotSameType(Object value) { - if (value == null) { - return false; - } - if (valueClass == null) { - valueClass = value.getClass(); - } - return valueClass != value.getClass(); + if (valueClass == null) { + valueClass = value.getClass(); } - - public void writeKey(Object key, MemoryBuffer buffer, ClassResolver classResolver, RefResolver refResolver, boolean trackingKeyRef, ClassInfoHolder keyClassInfoWriteCache) { - preserveByteForHeaderAndChunkSize(buffer); - updateKeyHeader(key, trackingKeyRef, keyIsNotSameType); - //todo hening key == null提到外面？ - if (!trackingKeyRef) { - if (key == null) { - buffer.writeByte(Fury.NULL_FLAG); - } else { - if (!keyIsNotSameType) { - writeKeyClass(key, buffer, keyClassInfoWriteCache); - keyClassInfoWriteCache.getSerializer().write(buffer, key); - } else { - fury.writeNonRef(buffer, key, classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache)); - } - } + return valueClass != value.getClass(); + } + + public void writeKey( + Object key, + MemoryBuffer buffer, + ClassResolver classResolver, + RefResolver refResolver, + boolean trackingKeyRef, + ClassInfoHolder keyClassInfoWriteCache) { + preserveByteForHeaderAndChunkSize(buffer); + updateKeyHeader(key, trackingKeyRef, keyIsNotSameType); + // todo hening key == null提到外面？ + if (!trackingKeyRef) { + if (key == null) { + buffer.writeByte(Fury.NULL_FLAG); + } else { + if (!keyIsNotSameType) { + writeKeyClass(key, buffer, keyClassInfoWriteCache); + keyClassInfoWriteCache.getSerializer().write(buffer, key); } else { - //todo 提到外面 - if (key == null) { - buffer.writeByte(Fury.NULL_FLAG); - } else { - if (!keyIsNotSameType) { - //todo key is not null, no need to write no null flag - writeKeyClass(key, buffer, keyClassInfoWriteCache); - fury.writeRef(buffer, key, keyClassInfoWriteCache.getSerializer()); - } else { - if (!refResolver.writeNullFlag(buffer, key)) { - fury.writeRef(buffer, key, classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache)); - } - } - } + fury.writeNonRef( + buffer, key, classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache)); } - - } - - public Object readKey(int header, MemoryBuffer memoryBuffer, ClassResolver classResolver, boolean trackingKeyRef, ClassInfoHolder keyClassInfoReadCache) { - this.header = header; - if (!trackingKeyRef) { - if (keyHasNull()) { - byte nullFlag = memoryBuffer.readByte(); - Preconditions.checkArgument(nullFlag == Fury.NULL_FLAG, "unexpected error"); - return null; - } else { - if (!keyIsNotSameType()) { - if (keySerializer == null) { - keySerializer = classResolver.readClassInfo(memoryBuffer, keyClassInfoReadCache).getSerializer(); - } - return keySerializer.read(memoryBuffer); - } else { - return fury.readNonRef(memoryBuffer, keyClassInfoReadCache); - } - } + } + } else { + // todo 提到外面 + if (key == null) { + buffer.writeByte(Fury.NULL_FLAG); + } else { + if (!keyIsNotSameType) { + // todo key is not null, no need to write no null flag + writeKeyClass(key, buffer, keyClassInfoWriteCache); + fury.writeRef(buffer, key, keyClassInfoWriteCache.getSerializer()); } else { - if (keyHasNull()) { - byte nullFlag = memoryBuffer.readByte(); - Preconditions.checkArgument(nullFlag == Fury.NULL_FLAG, "unexpected error"); - return null; - } else { - if (!keyIsNotSameType()) { - if (keySerializer == null) { - keySerializer = classResolver.readClassInfo(memoryBuffer, keyClassInfoReadCache).getSerializer(); - } - return fury.readRef(memoryBuffer, keySerializer); - } else { - return fury.readRef(memoryBuffer, keyClassInfoReadCache); - } - } + if (!refResolver.writeNullFlag(buffer, key)) { + fury.writeRef( + buffer, key, classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache)); + } } + } } - - - public void writeValue(Object value, MemoryBuffer buffer, ClassResolver classResolver, RefResolver refResolver, boolean trackingValueRef, ClassInfoHolder valueClassInfoWriteCache) { - preserveByteForHeaderAndChunkSize(buffer); - updateValueHeader(value, trackingValueRef, valueIsNotSameType); - if (!trackingValueRef) { - if (value == null) { - buffer.writeByte(Fury.NULL_FLAG); - } else { - if (!valueIsNotSameType) { - if (!valueHasNull()) { - writeValueClass(value, buffer, valueClassInfoWriteCache); - valueClassInfoWriteCache.getSerializer().write(buffer, value); - } else { - buffer.writeByte(Fury.NOT_NULL_VALUE_FLAG); - writeValueClass(value, buffer, valueClassInfoWriteCache); - valueClassInfoWriteCache.getSerializer().write(buffer, value); - } - } else { - fury.writeNullable(buffer, value, classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache)); - } - } + } + + public Object readKey( + int header, + MemoryBuffer memoryBuffer, + ClassResolver classResolver, + boolean trackingKeyRef, + ClassInfoHolder keyClassInfoReadCache) { + this.header = header; + if (!trackingKeyRef) { + if (keyHasNull()) { + byte nullFlag = memoryBuffer.readByte(); + Preconditions.checkArgument(nullFlag == Fury.NULL_FLAG, "unexpected error"); + return null; + } else { + if (!keyIsNotSameType()) { + if (keySerializer == null) { + keySerializer = + classResolver.readClassInfo(memoryBuffer, keyClassInfoReadCache).getSerializer(); + } + return keySerializer.read(memoryBuffer); } else { - if (value == null) { - buffer.writeByte(Fury.NULL_FLAG); - } else { - if (!valueIsNotSameType) { - writeValueClass(value, buffer, valueClassInfoWriteCache); - if (!valueHasNull()) { - fury.writeRef(buffer, value, valueClassInfoWriteCache.getSerializer()); - } else { - buffer.writeByte(Fury.NOT_NULL_VALUE_FLAG); - fury.writeRef(buffer, value, valueClassInfoWriteCache.getSerializer()); - } - } else { - if (!refResolver.writeNullFlag(buffer, value)) { - fury.writeRef(buffer, value, classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache)); - } - } - } + return fury.readNonRef(memoryBuffer, keyClassInfoReadCache); } - - } - - public Object readValue(int header, MemoryBuffer buffer, ClassResolver classResolver, boolean trackingValueRef, ClassInfoHolder valueClassInfoReadCache) { - this.header = header; - if (!trackingValueRef) { - if (!valueIsNotSameType()) { - if (valueHasNull()) { - byte flag = buffer.readByte(); - if (flag == Fury.NOT_NULL_VALUE_FLAG) { - if (valueSerializer == null) { - valueSerializer = classResolver.readClassInfo(buffer, valueClassInfoReadCache).getSerializer(); - } - return valueSerializer.read(buffer); - } else { - return null; - } - } else { - if (valueSerializer == null) { - valueSerializer = classResolver.readClassInfo(buffer, valueClassInfoReadCache).getSerializer(); - } - return valueSerializer.read(buffer); - } - } else { - return fury.readNullable(buffer, valueClassInfoReadCache); - } - + } + } else { + if (keyHasNull()) { + byte nullFlag = memoryBuffer.readByte(); + Preconditions.checkArgument(nullFlag == Fury.NULL_FLAG, "unexpected error"); + return null; + } else { + if (!keyIsNotSameType()) { + if (keySerializer == null) { + keySerializer = + classResolver.readClassInfo(memoryBuffer, keyClassInfoReadCache).getSerializer(); + } + return fury.readRef(memoryBuffer, keySerializer); } else { - if (!valueIsNotSameType()) { - if (valueHasNull()) { - byte flag = buffer.readByte(); - if (flag == Fury.NOT_NULL_VALUE_FLAG) { - if (valueSerializer == null) { - valueSerializer = classResolver.readClassInfo(buffer, valueClassInfoReadCache).getSerializer(); - } - return fury.readRef(buffer, valueSerializer); - } else { - return null; - } - } else { - if (valueSerializer == null) { - valueSerializer = classResolver.readClassInfo(buffer, valueClassInfoReadCache).getSerializer(); - } - return fury.readRef(buffer, valueSerializer); - } - } else { - return fury.readRef(buffer, valueClassInfoReadCache); - } + return fury.readRef(memoryBuffer, keyClassInfoReadCache); } + } } - - - private void updateKeyHeader(Object key, boolean trackingKeyRef, boolean keyIsNotSameType) { - if (key == null) { - header |= MapFlags.KEY_HAS_NULL; + } + + public void writeValue( + Object value, + MemoryBuffer buffer, + ClassResolver classResolver, + RefResolver refResolver, + boolean trackingValueRef, + ClassInfoHolder valueClassInfoWriteCache) { + preserveByteForHeaderAndChunkSize(buffer); + updateValueHeader(value, trackingValueRef, valueIsNotSameType); + if (!trackingValueRef) { + if (value == null) { + buffer.writeByte(Fury.NULL_FLAG); + } else { + if (!valueIsNotSameType) { + if (!valueHasNull()) { + writeValueClass(value, buffer, valueClassInfoWriteCache); + valueClassInfoWriteCache.getSerializer().write(buffer, value); + } else { + buffer.writeByte(Fury.NOT_NULL_VALUE_FLAG); + writeValueClass(value, buffer, valueClassInfoWriteCache); + valueClassInfoWriteCache.getSerializer().write(buffer, value); + } } else { - if (keyIsNotSameType) { - header |= MapFlags.KEY_NOT_SAME_TYPE; - } - if (trackingKeyRef) { - header |= MapFlags.TRACKING_KEY_REF; - } + fury.writeNullable( + buffer, + value, + classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache)); } - - } - - private void writeKeyClass(Object key, MemoryBuffer memoryBuffer, ClassInfoHolder keyClassInfoWriteCache) { - ClassResolver classResolver = fury.getClassResolver(); - ClassInfo classInfo = classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache); - if (!writeKeyClassInfo) { - classResolver.writeClass(memoryBuffer, classInfo); - writeKeyClassInfo = true; - } - } - - private void writeValueClass(Object value, MemoryBuffer memoryBuffer, ClassInfoHolder valueClassInfoWriteCache) { - ClassResolver classResolver = fury.getClassResolver(); - ClassInfo classInfo = classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache); - if (!writeValueClassInfo) { - classResolver.writeClass(memoryBuffer, classInfo); - writeValueClassInfo = true; - } - } - - private void updateFinalKeyHeader(Object key, boolean trackingKeyRef) { - if (trackingKeyRef) { - header |= MapFlags.TRACKING_KEY_REF; + } + } else { + if (value == null) { + buffer.writeByte(Fury.NULL_FLAG); + } else { + if (!valueIsNotSameType) { + writeValueClass(value, buffer, valueClassInfoWriteCache); + if (!valueHasNull()) { + fury.writeRef(buffer, value, valueClassInfoWriteCache.getSerializer()); + } else { + buffer.writeByte(Fury.NOT_NULL_VALUE_FLAG); + fury.writeRef(buffer, value, valueClassInfoWriteCache.getSerializer()); + } } else { - if (key == null) { - header |= MapFlags.KEY_HAS_NULL; - } + if (!refResolver.writeNullFlag(buffer, value)) { + fury.writeRef( + buffer, + value, + classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache)); + } } + } } - - private void updateFinalValueHeader(Object value, boolean trackingValueRef) { - if (trackingValueRef) { - header |= MapFlags.TRACKING_VALUE_REF; - } else { - if (value == null) { - header |= MapFlags.VALUE_HAS_NULL; + } + + public Object readValue( + int header, + MemoryBuffer buffer, + ClassResolver classResolver, + boolean trackingValueRef, + ClassInfoHolder valueClassInfoReadCache) { + this.header = header; + if (!trackingValueRef) { + if (!valueIsNotSameType()) { + if (valueHasNull()) { + byte flag = buffer.readByte(); + if (flag == Fury.NOT_NULL_VALUE_FLAG) { + if (valueSerializer == null) { + valueSerializer = + classResolver.readClassInfo(buffer, valueClassInfoReadCache).getSerializer(); } - } - } - - private void updateValueHeader(Object value, boolean trackingValueRef, boolean valueIsNotSameType) { - if (value == null) { - header |= MapFlags.VALUE_HAS_NULL; + return valueSerializer.read(buffer); + } else { + return null; + } } else { - if (valueIsNotSameType) { - header |= MapFlags.VALUE_NOT_SAME_TYPE; - } - if (trackingValueRef) { - header |= MapFlags.TRACKING_VALUE_REF; + if (valueSerializer == null) { + valueSerializer = + classResolver.readClassInfo(buffer, valueClassInfoReadCache).getSerializer(); + } + return valueSerializer.read(buffer); + } + } else { + return fury.readNullable(buffer, valueClassInfoReadCache); + } + + } else { + if (!valueIsNotSameType()) { + if (valueHasNull()) { + byte flag = buffer.readByte(); + if (flag == Fury.NOT_NULL_VALUE_FLAG) { + if (valueSerializer == null) { + valueSerializer = + classResolver.readClassInfo(buffer, valueClassInfoReadCache).getSerializer(); } + return fury.readRef(buffer, valueSerializer); + } else { + return null; + } + } else { + if (valueSerializer == null) { + valueSerializer = + classResolver.readClassInfo(buffer, valueClassInfoReadCache).getSerializer(); + } + return fury.readRef(buffer, valueSerializer); } + } else { + return fury.readRef(buffer, valueClassInfoReadCache); + } } - - /** - * update chunk size and header, if chunk size == 0, do nothing - * - * @param memoryBuffer memoryBuffer which is written - */ - public void writeHeader(MemoryBuffer memoryBuffer) { - if (chunkSize > 0) { - int currentWriteIndex = memoryBuffer.writerIndex(); - memoryBuffer.writerIndex(startOffset); - memoryBuffer.writeByte(chunkSize); - memoryBuffer.writeByte(header); - memoryBuffer.writerIndex(currentWriteIndex); - chunkSize = 0; - } + } + + private void updateKeyHeader(Object key, boolean trackingKeyRef, boolean keyIsNotSameType) { + if (key == null) { + header |= MapFlags.KEY_HAS_NULL; + } else { + if (keyIsNotSameType) { + header |= MapFlags.KEY_NOT_SAME_TYPE; + } + if (trackingKeyRef) { + header |= MapFlags.TRACKING_KEY_REF; + } } - - /** - * use chunk size = 0 to mark chunk write finish, - * if mark chunk write finish which means predict failed, chunk write is finish, - * rest of map will be written by generalJavaWrite - * - * @param memoryBuffer memoryBuffer which is written - */ - public void markChunkWriteFinish(MemoryBuffer memoryBuffer) { - if (!markChunkWriteFinish) { - writeHeader(memoryBuffer); - //set chunk size = 0 - memoryBuffer.writeByte(0); - markChunkWriteFinish = true; - } + } + + private void writeKeyClass( + Object key, MemoryBuffer memoryBuffer, ClassInfoHolder keyClassInfoWriteCache) { + ClassResolver classResolver = fury.getClassResolver(); + ClassInfo classInfo = classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache); + if (!writeKeyClassInfo) { + classResolver.writeClass(memoryBuffer, classInfo); + writeKeyClassInfo = true; } - - /** - * chunk size reach max size, start new chunk, no need reset keyClass and value Class - * - * @param memoryBuffer memoryBuffer which is written - */ - public void reset(MemoryBuffer memoryBuffer) { - writeHeader(memoryBuffer); - header = 0; - chunkSize = 0; - hasPreservedByte = false; - writeKeyClassInfo = false; - writeValueClassInfo = false; - prevKeyIsNull = false; - keyClass = null; - valueClass = null; + } + + private void writeValueClass( + Object value, MemoryBuffer memoryBuffer, ClassInfoHolder valueClassInfoWriteCache) { + ClassResolver classResolver = fury.getClassResolver(); + ClassInfo classInfo = classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache); + if (!writeValueClassInfo) { + classResolver.writeClass(memoryBuffer, classInfo); + writeValueClassInfo = true; } - - - private boolean keyHasNull() { - return (header & MapFlags.KEY_HAS_NULL) == MapFlags.KEY_HAS_NULL; + } + + private void updateFinalKeyHeader(Object key, boolean trackingKeyRef) { + if (trackingKeyRef) { + header |= MapFlags.TRACKING_KEY_REF; + } else { + if (key == null) { + header |= MapFlags.KEY_HAS_NULL; + } } - - private boolean valueHasNull() { - return (header & MapFlags.VALUE_HAS_NULL) == MapFlags.VALUE_HAS_NULL; + } + + private void updateFinalValueHeader(Object value, boolean trackingValueRef) { + if (trackingValueRef) { + header |= MapFlags.TRACKING_VALUE_REF; + } else { + if (value == null) { + header |= MapFlags.VALUE_HAS_NULL; + } } - - private boolean valueIsNotSameType() { - return (header & MapFlags.VALUE_NOT_SAME_TYPE) == MapFlags.VALUE_NOT_SAME_TYPE; + } + + private void updateValueHeader( + Object value, boolean trackingValueRef, boolean valueIsNotSameType) { + if (value == null) { + header |= MapFlags.VALUE_HAS_NULL; + } else { + if (valueIsNotSameType) { + header |= MapFlags.VALUE_NOT_SAME_TYPE; + } + if (trackingValueRef) { + header |= MapFlags.TRACKING_VALUE_REF; + } } - - private boolean keyIsNotSameType() { - return (header & MapFlags.KEY_NOT_SAME_TYPE) == MapFlags.KEY_NOT_SAME_TYPE; + } + + /** + * update chunk size and header, if chunk size == 0, do nothing + * + * @param memoryBuffer memoryBuffer which is written + */ + public void writeHeader(MemoryBuffer memoryBuffer) { + if (chunkSize > 0) { + int currentWriteIndex = memoryBuffer.writerIndex(); + memoryBuffer.writerIndex(startOffset); + memoryBuffer.writeByte(chunkSize); + memoryBuffer.writeByte(header); + memoryBuffer.writerIndex(currentWriteIndex); + chunkSize = 0; } - - public boolean isMarkChunkWriteFinish() { - return markChunkWriteFinish; + } + + /** + * use chunk size = 0 to mark chunk write finish, if mark chunk write finish which means predict + * failed, chunk write is finish, rest of map will be written by generalJavaWrite + * + * @param memoryBuffer memoryBuffer which is written + */ + public void markChunkWriteFinish(MemoryBuffer memoryBuffer) { + if (!markChunkWriteFinish) { + writeHeader(memoryBuffer); + // set chunk size = 0 + memoryBuffer.writeByte(0); + markChunkWriteFinish = true; } - - - public void setKeySerializer(Serializer keySerializer) { - this.keySerializer = keySerializer; - } - - public void setValueSerializer(Serializer valueSerializer) { - this.valueSerializer = valueSerializer; - } - + } + + /** + * chunk size reach max size, start new chunk, no need reset keyClass and value Class + * + * @param memoryBuffer memoryBuffer which is written + */ + public void reset(MemoryBuffer memoryBuffer) { + writeHeader(memoryBuffer); + header = 0; + chunkSize = 0; + hasPreservedByte = false; + writeKeyClassInfo = false; + writeValueClassInfo = false; + prevKeyIsNull = false; + keyClass = null; + valueClass = null; + } + + private boolean keyHasNull() { + return (header & MapFlags.KEY_HAS_NULL) == MapFlags.KEY_HAS_NULL; + } + + private boolean valueHasNull() { + return (header & MapFlags.VALUE_HAS_NULL) == MapFlags.VALUE_HAS_NULL; + } + + private boolean valueIsNotSameType() { + return (header & MapFlags.VALUE_NOT_SAME_TYPE) == MapFlags.VALUE_NOT_SAME_TYPE; + } + + private boolean keyIsNotSameType() { + return (header & MapFlags.KEY_NOT_SAME_TYPE) == MapFlags.KEY_NOT_SAME_TYPE; + } + + public boolean isMarkChunkWriteFinish() { + return markChunkWriteFinish; + } + + public void setKeySerializer(Serializer keySerializer) { + this.keySerializer = keySerializer; + } + + public void setValueSerializer(Serializer valueSerializer) { + this.valueSerializer = valueSerializer; + } } diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapFlags.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapFlags.java index 1c51a20a5c..52ca2d2175 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapFlags.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapFlags.java @@ -19,48 +19,31 @@ package org.apache.fury.serializer.collection; - public class MapFlags { - /** - * Whether track key ref. - */ - public static int TRACKING_KEY_REF = 0b1; - - /** - * Whether key has null. - */ - public static int KEY_HAS_NULL = 0b10; - -// /** -// * Whether key is not declare type. -// */ -// public static int KEY_NOT_DECL_TYPE = 0b100; + /** Whether track key ref. */ + public static int TRACKING_KEY_REF = 0b1; - /** - * Whether keys type are different. - */ - public static int KEY_NOT_SAME_TYPE = 0b100; + /** Whether key has null. */ + public static int KEY_HAS_NULL = 0b10; - /** - * Whether track value ref. - */ - public static int TRACKING_VALUE_REF = 0b1000; + // /** + // * Whether key is not declare type. + // */ + // public static int KEY_NOT_DECL_TYPE = 0b100; - /** - * Whether value has null. - */ - public static int VALUE_HAS_NULL = 0b10000; + /** Whether keys type are different. */ + public static int KEY_NOT_SAME_TYPE = 0b100; - /** - * Whether value is not declare type. - */ -// public static int VALUE_NOT_DECL_TYPE = 0b1000000; + /** Whether track value ref. */ + public static int TRACKING_VALUE_REF = 0b1000; - /** - * Whether values type are different. - */ - public static int VALUE_NOT_SAME_TYPE = 0b100000; + /** Whether value has null. */ + public static int VALUE_HAS_NULL = 0b10000; + /** Whether value is not declare type. */ + // public static int VALUE_NOT_DECL_TYPE = 0b1000000; + /** Whether values type are different. */ + public static int VALUE_NOT_SAME_TYPE = 0b100000; } diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapSerializer.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapSerializer.java index bcae76116d..3eaea0abcb 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapSerializer.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapSerializer.java @@ -44,9 +44,9 @@ public Map onMapWrite(MemoryBuffer buffer, T value) { public T read(MemoryBuffer buffer) { Map map = newMap(buffer); if (fury.isChunkSerializeMapEnabled()) { - chunkReadElements(buffer, getAndClearNumElements(), map); + chunkReadElements(buffer, getAndClearNumElements(), map); } else { - readElements(buffer, getAndClearNumElements(), map); + readElements(buffer, getAndClearNumElements(), map); } return onMapRead(map); } diff --git a/java/fury-core/src/test/java/org/apache/fury/serializer/collection/MapSerializersTest.java b/java/fury-core/src/test/java/org/apache/fury/serializer/collection/MapSerializersTest.java index c97f3790b2..1447cb9ffe 100644 --- a/java/fury-core/src/test/java/org/apache/fury/serializer/collection/MapSerializersTest.java +++ b/java/fury-core/src/test/java/org/apache/fury/serializer/collection/MapSerializersTest.java @@ -365,30 +365,30 @@ public void testStringKeyMapSerializer() { } } - @Test(dataProvider = "javaFury") - public void testDifferentKeyAndValueType(Fury fury) { - Map map = createDifferentKeyAndValueTypeMap(); - Assert.assertEquals(serDe(fury, map), map); - } + @Test(dataProvider = "javaFury") + public void testDifferentKeyAndValueType(Fury fury) { + Map map = createDifferentKeyAndValueTypeMap(); + Assert.assertEquals(serDe(fury, map), map); + } - private static Map createDifferentKeyAndValueTypeMap() { - Map map = new HashMap<>(); - map.put(null, "1"); - map.put(2, "1"); - map.put(4, "1"); - map.put(6, "1"); - map.put(7, "1"); - map.put(10, "1"); - map.put(12, "null"); - map.put(19, "null"); - map.put(11, null); - map.put(20, null); - map.put(21, 9); - map.put(22, 99); - map.put(291, 900); - map.put("292", 900); - map.put("293", 900); - map.put("23", 900); - return map; - } + private static Map createDifferentKeyAndValueTypeMap() { + Map map = new HashMap<>(); + map.put(null, "1"); + map.put(2, "1"); + map.put(4, "1"); + map.put(6, "1"); + map.put(7, "1"); + map.put(10, "1"); + map.put(12, "null"); + map.put(19, "null"); + map.put(11, null); + map.put(20, null); + map.put(21, 9); + map.put(22, 99); + map.put(291, 900); + map.put("292", 900); + map.put("293", 900); + map.put("23", 900); + return map; + } } From 55f8440cbb1a9423241efcf72d25fe18fbbddef1 Mon Sep 17 00:00:00 2001 From: hening Date: Fri, 2 Aug 2024 11:07:55 +0800 Subject: [PATCH 14/18] reset key and value serializer is null --- .../org/apache/fury/serializer/collection/MapChunkWriter.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapChunkWriter.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapChunkWriter.java index 806189d536..b45b8bf752 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapChunkWriter.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapChunkWriter.java @@ -517,6 +517,8 @@ public void reset(MemoryBuffer memoryBuffer) { prevKeyIsNull = false; keyClass = null; valueClass = null; + keySerializer = null; + valueSerializer = null; } private boolean keyHasNull() { From 37523395dfc0421381f553f5e437a3308ca84318 Mon Sep 17 00:00:00 2001 From: hening Date: Tue, 6 Aug 2024 17:39:30 +0800 Subject: [PATCH 15/18] modify code --- .../collection/AbstractMapSerializer.java | 60 +++--- .../serializer/collection/MapChunkWriter.java | 174 ++++++++++++++---- 2 files changed, 179 insertions(+), 55 deletions(-) diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java index 5e824da958..0389fd6575 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java @@ -154,23 +154,24 @@ protected final void chunkWriteElements(Fury fury, MemoryBuffer buffer, Map map) this.keySerializer = null; this.valueSerializer = null; if (keySerializer != null && valueSerializer != null) { - javaWriteWithKVSerializers(fury, buffer, map, keySerializer, valueSerializer); + javaChunkWriteWithKVSerializers(fury, buffer, map, keySerializer, valueSerializer); } else if (keySerializer != null) { - javaWriteWithKeySerializers(map, buffer, keySerializer); + javaChunkWriteWithKeySerializers(map, buffer, keySerializer); } else if (valueSerializer != null) { - javaWriteWithValueSerializers(map, buffer, valueSerializer); + javaChunkWriteWithValueSerializers(map, buffer, valueSerializer); } else { genericJavaChunkWrite(fury, buffer, map); } } - private void javaWriteWithKeySerializers(Map map, MemoryBuffer buffer, Serializer keySerializer) { + private void javaChunkWriteWithKeySerializers( + Map map, MemoryBuffer buffer, Serializer keySerializer) { MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); ClassResolver classResolver = fury.getClassResolver(); RefResolver refResolver = fury.getRefResolver(); for (Object object : map.entrySet()) { Map.Entry entry = (Map.Entry) object; - mapChunkWriter = mapChunkWriter.next(entry.getKey(), entry.getValue(), buffer); + mapChunkWriter = mapChunkWriter.finalKeyNext(entry.getKey(), entry.getValue(), buffer); mapChunkWriter.writeFinalKey(entry.getKey(), buffer, keySerializer); Object value = entry.getValue(); mapChunkWriter.writeValue( @@ -180,7 +181,7 @@ private void javaWriteWithKeySerializers(Map map, MemoryBuffer buffer, Serialize mapChunkWriter.writeHeader(buffer); } - private void javaWriteWithValueSerializers( + private void javaChunkWriteWithValueSerializers( Map map, MemoryBuffer buffer, Serializer valueSerializer) { MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); ClassResolver classResolver = fury.getClassResolver(); @@ -188,7 +189,7 @@ private void javaWriteWithValueSerializers( for (Object object : map.entrySet()) { Map.Entry entry = (Map.Entry) object; Object key = entry.getKey(); - mapChunkWriter = mapChunkWriter.next(entry.getKey(), entry.getValue(), buffer); + mapChunkWriter = mapChunkWriter.finalValueNext(entry.getKey(), entry.getValue(), buffer); mapChunkWriter.writeKey( key, buffer, classResolver, refResolver, fury.trackingRef(), keyClassInfoWriteCache); mapChunkWriter.writeFinalValue(entry.getValue(), buffer, valueSerializer); @@ -197,7 +198,7 @@ private void javaWriteWithValueSerializers( mapChunkWriter.writeHeader(buffer); } - private void javaWriteWithKVSerializers( + private void javaChunkWriteWithKVSerializers( Fury fury, MemoryBuffer buffer, Map map, @@ -208,7 +209,7 @@ private void javaWriteWithKVSerializers( Map.Entry entry = (Map.Entry) object; Object key = entry.getKey(); Object value = entry.getValue(); - mapChunkWriter = mapChunkWriter.next(entry.getKey(), entry.getValue(), buffer); + mapChunkWriter = mapChunkWriter.finalKVNext(entry.getKey(), entry.getValue(), buffer); mapChunkWriter.writeFinalKey(key, buffer, keySerializer); mapChunkWriter.writeFinalValue(value, buffer, valueSerializer); mapChunkWriter.increaseChunkSize(); @@ -216,6 +217,21 @@ private void javaWriteWithKVSerializers( mapChunkWriter.writeHeader(buffer); } + private void javaWriteWithKVSerializers( + Fury fury, + MemoryBuffer buffer, + Map map, + Serializer keySerializer, + Serializer valueSerializer) { + for (Object object : map.entrySet()) { + Map.Entry entry = (Map.Entry) object; + Object key = entry.getKey(); + Object value = entry.getValue(); + fury.writeRef(buffer, key, keySerializer); + fury.writeRef(buffer, value, valueSerializer); + } + } + private void genericJavaWrite(Fury fury, MemoryBuffer buffer, Map map) { Generics generics = fury.getGenerics(); GenericType genericType = generics.nextGenericType(); @@ -355,7 +371,7 @@ private void javaKVTypesFinalChunkWrite( Map.Entry entry = (Map.Entry) object; Object key = entry.getKey(); Object value = entry.getValue(); - mapChunkWriter = mapChunkWriter.next(key, value, buffer); + mapChunkWriter = mapChunkWriter.finalKVNext(key, value, buffer); generics.pushGenericType(keyGenericType); mapChunkWriter.writeFinalKey(key, buffer, keySerializer); generics.popGenericType(); @@ -412,7 +428,7 @@ private void javaKeyTypeFinalChunkWrite( Map.Entry entry = (Map.Entry) object; Object key = entry.getKey(); Object value = entry.getValue(); - mapChunkWriter = mapChunkWriter.next(key, value, buffer); + mapChunkWriter = mapChunkWriter.finalKeyNext(key, value, buffer); generics.pushGenericType(keyGenericType); mapChunkWriter.writeFinalKey(key, buffer, keySerializer); generics.popGenericType(); @@ -441,7 +457,7 @@ private void javaValueTypeFinalChunkWrite( Map.Entry entry = (Map.Entry) object; Object key = entry.getKey(); Object value = entry.getValue(); - mapChunkWriter = mapChunkWriter.next(key, value, buffer); + mapChunkWriter = mapChunkWriter.finalValueNext(key, value, buffer); generics.pushGenericType(keyGenericType); mapChunkWriter.writeKey( key, buffer, classResolver, refResolver, trackingKeyRef, keyClassInfoWriteCache); @@ -885,8 +901,8 @@ private void javaKVTypesFinalChunkRead( while (size > 0) { byte chunkSize = buffer.readByte(); byte header = buffer.readByte(); - mapChunkWriter.setKeySerializer(null); - mapChunkWriter.setValueSerializer(null); + mapChunkWriter.setKeyReadSerializer(null); + mapChunkWriter.setValueReadSerializer(null); Preconditions.checkArgument( chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); @@ -944,8 +960,8 @@ private void javaKeyTypeFinalChunkRead( chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); byte header = buffer.readByte(); - mapChunkWriter.setKeySerializer(null); - mapChunkWriter.setValueSerializer(null); + mapChunkWriter.setKeyReadSerializer(null); + mapChunkWriter.setValueReadSerializer(null); while (chunkSize > 0) { generics.pushGenericType(keyGenericType); Object key = mapChunkWriter.readFinalKey(buffer, header, keySerializer); @@ -1004,8 +1020,8 @@ private void javaValueTypeFinalChunkRead( chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); byte header = buffer.readByte(); - mapChunkWriter.setKeySerializer(null); - mapChunkWriter.setValueSerializer(null); + mapChunkWriter.setKeyReadSerializer(null); + mapChunkWriter.setValueReadSerializer(null); while (chunkSize > 0) { generics.pushGenericType(keyGenericType); Object key = @@ -1080,8 +1096,8 @@ private void javaKVTypesNonFinalChunkRead( } } else { byte header = buffer.readByte(); - mapChunkWriter.setKeySerializer(null); - mapChunkWriter.setValueSerializer(null); + mapChunkWriter.setKeyReadSerializer(null); + mapChunkWriter.setValueReadSerializer(null); while (chunkSize > 0) { generics.pushGenericType(keyGenericType); Object key = @@ -1145,8 +1161,8 @@ private void generalJavaChunkRead(Fury fury, MemoryBuffer buffer, Map map, int s } } else { byte header = buffer.readByte(); - mapChunkWriter.setKeySerializer(null); - mapChunkWriter.setValueSerializer(null); + mapChunkWriter.setKeyReadSerializer(null); + mapChunkWriter.setValueReadSerializer(null); while (chunkSize > 0) { Object key = mapChunkWriter.readKey( diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapChunkWriter.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapChunkWriter.java index b45b8bf752..8466a9aef5 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapChunkWriter.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapChunkWriter.java @@ -28,8 +28,10 @@ public MapChunkWriter(Fury fury) { private boolean keyIsNotSameType = false; private boolean valueIsNotSameType = false; private boolean prevKeyIsNull = false; - private Serializer keySerializer; - private Serializer valueSerializer; + private Serializer keyReadSerializer; + private Serializer valueReadSerializer; + private Serializer keyWriteSerializer; + private Serializer valueWriteSerializer; /** mark chunk write finish */ private boolean markChunkWriteFinish = false; @@ -51,6 +53,78 @@ private void preserveByteForHeaderAndChunkSize(MemoryBuffer memoryBuffer) { hasPreservedByte = true; } + public MapChunkWriter finalKVNext(Object key, Object value, MemoryBuffer buffer) { + if (key == null) { + prevKeyIsNull = true; + if (chunkSize > 0) { + reset(buffer); + } + } + if (prevKeyIsNull && key != null) { + reset(buffer); + } + if (value == null && chunkSize > 0 && !valueHasNull()) { + // if value has null before, no need to reset chunk + reset(buffer); + } + if (chunkSize >= MAX_CHUNK_SIZE) { + reset(buffer); + } + return this; + } + + public MapChunkWriter finalKeyNext(Object key, Object value, MemoryBuffer buffer) { + if (key == null) { + prevKeyIsNull = true; + if (chunkSize > 0) { + reset(buffer); + } + } + if (prevKeyIsNull && key != null) { + reset(buffer); + } + if (value == null && chunkSize > 0 && !valueHasNull()) { + // if value has null before, no need to reset chunk + reset(buffer); + } + if (!valueIsNotSameType) { + this.valueIsNotSameType = judgeValueIsNotSameType(value); + if (valueIsNotSameType) { + reset(buffer); + } + } + if (chunkSize >= MAX_CHUNK_SIZE) { + reset(buffer); + } + return this; + } + + public MapChunkWriter finalValueNext(Object key, Object value, MemoryBuffer buffer) { + if (key == null) { + prevKeyIsNull = true; + if (chunkSize > 0) { + reset(buffer); + } + } + if (prevKeyIsNull && key != null) { + reset(buffer); + } + if (value == null && chunkSize > 0 && !valueHasNull()) { + // if value has null before, no need to reset chunk + reset(buffer); + } + if (!keyIsNotSameType) { + this.keyIsNotSameType = judgeKeyIsNotSameType(key); + if (keyIsNotSameType) { + reset(buffer); + } + } + if (chunkSize >= MAX_CHUNK_SIZE) { + reset(buffer); + } + return this; + } + public MapChunkWriter next(Object key, Object value, MemoryBuffer buffer) { if (!markChunkWriteFinish) { if (key == null) { @@ -227,7 +301,7 @@ public void writeKey( } else { if (!keyIsNotSameType) { writeKeyClass(key, buffer, keyClassInfoWriteCache); - keyClassInfoWriteCache.getSerializer().write(buffer, key); + keyWriteSerializer.write(buffer, key); } else { fury.writeNonRef( buffer, key, classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache)); @@ -241,7 +315,7 @@ public void writeKey( if (!keyIsNotSameType) { // todo key is not null, no need to write no null flag writeKeyClass(key, buffer, keyClassInfoWriteCache); - fury.writeRef(buffer, key, keyClassInfoWriteCache.getSerializer()); + writeNoNullRef(keyWriteSerializer, key, buffer, refResolver); } else { if (!refResolver.writeNullFlag(buffer, key)) { fury.writeRef( @@ -266,11 +340,11 @@ public Object readKey( return null; } else { if (!keyIsNotSameType()) { - if (keySerializer == null) { - keySerializer = + if (keyReadSerializer == null) { + keyReadSerializer = classResolver.readClassInfo(memoryBuffer, keyClassInfoReadCache).getSerializer(); } - return keySerializer.read(memoryBuffer); + return keyReadSerializer.read(memoryBuffer); } else { return fury.readNonRef(memoryBuffer, keyClassInfoReadCache); } @@ -282,11 +356,11 @@ public Object readKey( return null; } else { if (!keyIsNotSameType()) { - if (keySerializer == null) { - keySerializer = + if (keyReadSerializer == null) { + keyReadSerializer = classResolver.readClassInfo(memoryBuffer, keyClassInfoReadCache).getSerializer(); } - return fury.readRef(memoryBuffer, keySerializer); + return readNoNullRef(keyReadSerializer, memoryBuffer); } else { return fury.readRef(memoryBuffer, keyClassInfoReadCache); } @@ -294,6 +368,33 @@ public Object readKey( } } + private void writeNoNullRef( + Serializer serializer, Object o, MemoryBuffer buffer, RefResolver refResolver) { + if (serializer.needToWriteRef()) { + if (!refResolver.writeRefOrNull(buffer, o)) { + serializer.write(buffer, o); + } + } else { + serializer.write(buffer, o); + } + } + + private Object readNoNullRef(Serializer serializer, MemoryBuffer memoryBuffer) { + if (serializer.needToWriteRef()) { + final RefResolver refResolver = fury.getRefResolver(); + int nextReadRefId = refResolver.tryPreserveRefId(memoryBuffer); + if (nextReadRefId >= Fury.NOT_NULL_VALUE_FLAG) { + Object obj = serializer.read(memoryBuffer); + refResolver.setReadObject(nextReadRefId, obj); + return obj; + } else { + return refResolver.getReadObject(); + } + } else { + return serializer.read(memoryBuffer); + } + } + public void writeValue( Object value, MemoryBuffer buffer, @@ -310,11 +411,11 @@ public void writeValue( if (!valueIsNotSameType) { if (!valueHasNull()) { writeValueClass(value, buffer, valueClassInfoWriteCache); - valueClassInfoWriteCache.getSerializer().write(buffer, value); + valueWriteSerializer.write(buffer, value); } else { buffer.writeByte(Fury.NOT_NULL_VALUE_FLAG); writeValueClass(value, buffer, valueClassInfoWriteCache); - valueClassInfoWriteCache.getSerializer().write(buffer, value); + valueWriteSerializer.write(buffer, value); } } else { fury.writeNullable( @@ -330,10 +431,9 @@ public void writeValue( if (!valueIsNotSameType) { writeValueClass(value, buffer, valueClassInfoWriteCache); if (!valueHasNull()) { - fury.writeRef(buffer, value, valueClassInfoWriteCache.getSerializer()); + writeNoNullRef(valueWriteSerializer, value, buffer, refResolver); } else { - buffer.writeByte(Fury.NOT_NULL_VALUE_FLAG); - fury.writeRef(buffer, value, valueClassInfoWriteCache.getSerializer()); + fury.writeRef(buffer, value, valueWriteSerializer); } } else { if (!refResolver.writeNullFlag(buffer, value)) { @@ -359,20 +459,20 @@ public Object readValue( if (valueHasNull()) { byte flag = buffer.readByte(); if (flag == Fury.NOT_NULL_VALUE_FLAG) { - if (valueSerializer == null) { - valueSerializer = + if (valueReadSerializer == null) { + valueReadSerializer = classResolver.readClassInfo(buffer, valueClassInfoReadCache).getSerializer(); } - return valueSerializer.read(buffer); + return valueReadSerializer.read(buffer); } else { return null; } } else { - if (valueSerializer == null) { - valueSerializer = + if (valueReadSerializer == null) { + valueReadSerializer = classResolver.readClassInfo(buffer, valueClassInfoReadCache).getSerializer(); } - return valueSerializer.read(buffer); + return valueReadSerializer.read(buffer); } } else { return fury.readNullable(buffer, valueClassInfoReadCache); @@ -383,20 +483,20 @@ public Object readValue( if (valueHasNull()) { byte flag = buffer.readByte(); if (flag == Fury.NOT_NULL_VALUE_FLAG) { - if (valueSerializer == null) { - valueSerializer = + if (valueReadSerializer == null) { + valueReadSerializer = classResolver.readClassInfo(buffer, valueClassInfoReadCache).getSerializer(); } - return fury.readRef(buffer, valueSerializer); + return fury.readRef(buffer, valueReadSerializer); } else { return null; } } else { - if (valueSerializer == null) { - valueSerializer = + if (valueReadSerializer == null) { + valueReadSerializer = classResolver.readClassInfo(buffer, valueClassInfoReadCache).getSerializer(); } - return fury.readRef(buffer, valueSerializer); + return readNoNullRef(valueReadSerializer, buffer); } } else { return fury.readRef(buffer, valueClassInfoReadCache); @@ -425,6 +525,9 @@ private void writeKeyClass( classResolver.writeClass(memoryBuffer, classInfo); writeKeyClassInfo = true; } + if (keyWriteSerializer == null) { + keyWriteSerializer = classInfo.getSerializer(); + } } private void writeValueClass( @@ -435,6 +538,9 @@ private void writeValueClass( classResolver.writeClass(memoryBuffer, classInfo); writeValueClassInfo = true; } + if (valueWriteSerializer == null) { + valueWriteSerializer = classInfo.getSerializer(); + } } private void updateFinalKeyHeader(Object key, boolean trackingKeyRef) { @@ -517,8 +623,10 @@ public void reset(MemoryBuffer memoryBuffer) { prevKeyIsNull = false; keyClass = null; valueClass = null; - keySerializer = null; - valueSerializer = null; + keyReadSerializer = null; + valueReadSerializer = null; + keyWriteSerializer = null; + valueWriteSerializer = null; } private boolean keyHasNull() { @@ -541,11 +649,11 @@ public boolean isMarkChunkWriteFinish() { return markChunkWriteFinish; } - public void setKeySerializer(Serializer keySerializer) { - this.keySerializer = keySerializer; + public void setKeyReadSerializer(Serializer keyReadSerializer) { + this.keyReadSerializer = keyReadSerializer; } - public void setValueSerializer(Serializer valueSerializer) { - this.valueSerializer = valueSerializer; + public void setValueReadSerializer(Serializer valueReadSerializer) { + this.valueReadSerializer = valueReadSerializer; } } From 48b8a7539e9ebe97f5a03b5d64a94cac68902d1e Mon Sep 17 00:00:00 2001 From: hening Date: Wed, 7 Aug 2024 10:03:26 +0800 Subject: [PATCH 16/18] add apache license --- .../serializer/collection/MapChunkWriter.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapChunkWriter.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapChunkWriter.java index 8466a9aef5..1e6fa1411c 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapChunkWriter.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/MapChunkWriter.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.fury.serializer.collection; import org.apache.fury.Fury; From c06fdc7b70f714e5643e7d7ac161c68663d6265d Mon Sep 17 00:00:00 2001 From: hening Date: Mon, 9 Sep 2024 20:30:44 +0800 Subject: [PATCH 17/18] use local variable and bug fix --- .../collection/AbstractMapSerializer.java | 1465 +++++++++++++++-- 1 file changed, 1340 insertions(+), 125 deletions(-) diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java index 0389fd6575..216085f68a 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java @@ -29,6 +29,7 @@ import org.apache.fury.memory.MemoryBuffer; import org.apache.fury.reflect.ReflectionUtils; import org.apache.fury.reflect.TypeRef; +import org.apache.fury.resolver.ClassInfo; import org.apache.fury.resolver.ClassInfoHolder; import org.apache.fury.resolver.ClassResolver; import org.apache.fury.resolver.RefResolver; @@ -41,6 +42,7 @@ /** Serializer for all map-like objects. */ @SuppressWarnings({"unchecked", "rawtypes"}) public abstract class AbstractMapSerializer extends Serializer { + private static final int MAX_CHUNK_SIZE = 127; protected MethodHandle constructor; protected final boolean supportCodegenHook; private Serializer keySerializer; @@ -154,7 +156,7 @@ protected final void chunkWriteElements(Fury fury, MemoryBuffer buffer, Map map) this.keySerializer = null; this.valueSerializer = null; if (keySerializer != null && valueSerializer != null) { - javaChunkWriteWithKVSerializers(fury, buffer, map, keySerializer, valueSerializer); + javaChunkWriteWithKVSerializers(buffer, map, keySerializer, valueSerializer); } else if (keySerializer != null) { javaChunkWriteWithKeySerializers(map, buffer, keySerializer); } else if (valueSerializer != null) { @@ -166,55 +168,414 @@ protected final void chunkWriteElements(Fury fury, MemoryBuffer buffer, Map map) private void javaChunkWriteWithKeySerializers( Map map, MemoryBuffer buffer, Serializer keySerializer) { - MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); ClassResolver classResolver = fury.getClassResolver(); RefResolver refResolver = fury.getRefResolver(); + boolean prevKeyIsNull = false; + int header = 0; + int chunkSize = 0; + int startOffset = 0; + boolean hasPreservedByte = false; + boolean valueIsNotSameType = false; + Class valueClass = null; + Serializer valueWriteSerializer = null; + boolean writeValueClassInfo = false; + boolean needReset = false; for (Object object : map.entrySet()) { Map.Entry entry = (Map.Entry) object; - mapChunkWriter = mapChunkWriter.finalKeyNext(entry.getKey(), entry.getValue(), buffer); - mapChunkWriter.writeFinalKey(entry.getKey(), buffer, keySerializer); - Object value = entry.getValue(); - mapChunkWriter.writeValue( - value, buffer, classResolver, refResolver, fury.trackingRef(), valueClassInfoWriteCache); - mapChunkWriter.increaseChunkSize(); + Object key = entry.getKey(); + final Object value = entry.getValue(); + if (key == null) { + prevKeyIsNull = true; + } + if (!valueIsNotSameType) { + if (value != null) { + if (valueClass == null) { + valueClass = value.getClass(); + } + valueIsNotSameType = valueClass != value.getClass(); + if (valueIsNotSameType) { + needReset = true; + } + } + } + if ((key == null && chunkSize > 0) + || (prevKeyIsNull && key != null) + || (value == null && chunkSize > 0 && !valueHasNull(header)) + || needReset + || (chunkSize >= MAX_CHUNK_SIZE)) { + writeHeader(buffer, chunkSize, header, startOffset); + prevKeyIsNull = false; + header = 0; + chunkSize = 0; + startOffset = 0; + hasPreservedByte = false; + valueClass = value == null ? null : value.getClass(); + valueWriteSerializer = null; + writeValueClassInfo = false; + needReset = false; + } + if (!hasPreservedByte) { + int writerIndex = buffer.writerIndex(); + // preserve two byte for header and chunk size + buffer.writerIndex(writerIndex + 2); + startOffset = writerIndex; + hasPreservedByte = true; + } + // write final key + boolean trackingKeyRef = keySerializer.needToWriteRef(); + boolean trackingValueRef = fury.trackingRef(); + if (!valueIsNotSameType && valueWriteSerializer == null && !writeValueClassInfo) { + valueWriteSerializer = getSerializer(value, valueClassInfoWriteCache); + } + header = + updateKVHeader( + key, trackingKeyRef, value, trackingValueRef, header, false, valueIsNotSameType); + writeFinalKey(key, buffer, keySerializer, trackingKeyRef); + if (!writeValueClassInfo && valueWriteSerializer != null) { + writeValue( + value, + buffer, + valueWriteSerializer, + classResolver, + refResolver, + trackingValueRef, + valueIsNotSameType, + header, + true); + writeValueClassInfo = true; + } else { + writeValue( + value, + buffer, + valueWriteSerializer, + classResolver, + refResolver, + trackingValueRef, + valueIsNotSameType, + header, + false); + } + chunkSize++; + } + writeHeader(buffer, chunkSize, header, startOffset); + } + + private void writeKey( + Object key, + MemoryBuffer buffer, + Serializer keyWriteSerializer, + ClassResolver classResolver, + RefResolver refResolver, + boolean trackingKeyRef, + boolean keyIsNotSameType, + boolean needWriteClass) { + if (!trackingKeyRef) { + if (key == null) { + buffer.writeByte(Fury.NULL_FLAG); + } else { + if (!keyIsNotSameType) { + writeClass(key, buffer, valueClassInfoWriteCache, needWriteClass); + keyWriteSerializer.write(buffer, key); + } else { + fury.writeNonRef( + buffer, key, classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache)); + } + } + } else { + // todo 提到外面 + if (key == null) { + buffer.writeByte(Fury.NULL_FLAG); + } else { + if (!keyIsNotSameType) { + // todo key is not null, no need to write no null flag + writeClass(key, buffer, valueClassInfoWriteCache, needWriteClass); + writeNoNullRef(keyWriteSerializer, key, buffer, refResolver); + } else { + if (!refResolver.writeNullFlag(buffer, key)) { + fury.writeRef( + buffer, key, classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache)); + } + } + } + } + } + + private void writeValue( + Object value, + MemoryBuffer buffer, + Serializer valueWriteSerializer, + ClassResolver classResolver, + RefResolver refResolver, + boolean trackingValueRef, + boolean valueIsNotSameType, + int header, + boolean needWriteClass) { + if (!trackingValueRef) { + if (value == null) { + buffer.writeByte(Fury.NULL_FLAG); + } else { + if (!valueIsNotSameType) { + if (valueHasNull(header)) { + buffer.writeByte(Fury.NOT_NULL_VALUE_FLAG); + } + writeClass(value, buffer, valueClassInfoWriteCache, needWriteClass); + valueWriteSerializer.write(buffer, value); + } else { + fury.writeNullable( + buffer, + value, + classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache)); + } + } + } else { + if (value == null) { + buffer.writeByte(Fury.NULL_FLAG); + } else { + if (!valueIsNotSameType) { + writeClass(value, buffer, valueClassInfoWriteCache, needWriteClass); + if (!valueHasNull(header)) { + writeNoNullRef(valueWriteSerializer, value, buffer, refResolver); + } else { + fury.writeRef(buffer, value, valueWriteSerializer); + } + } else { + if (!refResolver.writeNullFlag(buffer, value)) { + fury.writeRef( + buffer, + value, + classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache)); + } + } + } + } + } + + private void writeClass( + Object o, + MemoryBuffer memoryBuffer, + ClassInfoHolder classInfoWriteCache, + boolean needWriteClass) { + ClassResolver classResolver = fury.getClassResolver(); + ClassInfo classInfo = classResolver.getClassInfo(o.getClass(), classInfoWriteCache); + if (needWriteClass) { + classResolver.writeClass(memoryBuffer, classInfo); } - mapChunkWriter.writeHeader(buffer); } private void javaChunkWriteWithValueSerializers( Map map, MemoryBuffer buffer, Serializer valueSerializer) { - MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); ClassResolver classResolver = fury.getClassResolver(); RefResolver refResolver = fury.getRefResolver(); + boolean prevKeyIsNull = false; + int header = 0; + int chunkSize = 0; + int startOffset = 0; + boolean hasPreservedByte = false; + boolean keyIsNotSameType = false; + Class keyClass = null; + Serializer keyWriteSerializer = null; + boolean writeKeyClassInfo = false; + boolean needReset = false; for (Object object : map.entrySet()) { Map.Entry entry = (Map.Entry) object; Object key = entry.getKey(); - mapChunkWriter = mapChunkWriter.finalValueNext(entry.getKey(), entry.getValue(), buffer); - mapChunkWriter.writeKey( - key, buffer, classResolver, refResolver, fury.trackingRef(), keyClassInfoWriteCache); - mapChunkWriter.writeFinalValue(entry.getValue(), buffer, valueSerializer); - mapChunkWriter.increaseChunkSize(); + final Object value = entry.getValue(); + if (key == null) { + prevKeyIsNull = true; + } + if (!keyIsNotSameType) { + if (key != null) { + if (keyClass == null) { + keyClass = key.getClass(); + } + keyIsNotSameType = keyClass != key.getClass(); + if (keyIsNotSameType) { + needReset = true; + } + } + } + if ((key == null && chunkSize > 0) + || (prevKeyIsNull && key != null) + || (value == null && chunkSize > 0 && !valueHasNull(header)) + || needReset + || (chunkSize >= MAX_CHUNK_SIZE)) { + writeHeader(buffer, chunkSize, header, startOffset); + prevKeyIsNull = false; + header = 0; + chunkSize = 0; + startOffset = 0; + hasPreservedByte = false; + keyClass = key == null ? null : key.getClass(); + keyWriteSerializer = null; + writeKeyClassInfo = false; + } + if (!hasPreservedByte) { + int writerIndex = buffer.writerIndex(); + // preserve two byte for header and chunk size + buffer.writerIndex(writerIndex + 2); + startOffset = writerIndex; + hasPreservedByte = true; + } + boolean trackingKeyRef = fury.trackingRef(); + boolean trackingValueRef = valueSerializer.needToWriteRef(); + if (!keyIsNotSameType && keyWriteSerializer == null && !writeKeyClassInfo) { + keyWriteSerializer = getSerializer(key, keyClassInfoWriteCache); + } + header = + updateKVHeader( + key, trackingKeyRef, value, trackingValueRef, header, keyIsNotSameType, false); + if (!writeKeyClassInfo && key != null) { + writeKey( + key, + buffer, + keyWriteSerializer, + classResolver, + refResolver, + trackingKeyRef, + keyIsNotSameType, + true); + writeKeyClassInfo = true; + } else { + writeKey( + key, + buffer, + keyWriteSerializer, + classResolver, + refResolver, + trackingKeyRef, + keyIsNotSameType, + false); + } + writeFinalValue(value, buffer, valueSerializer, trackingValueRef, header); + chunkSize++; + } + writeHeader(buffer, chunkSize, header, startOffset); + } + + private Serializer getSerializer(Object o, ClassInfoHolder classInfoWriteCache) { + if (o != null) { + ClassResolver classResolver = fury.getClassResolver(); + ClassInfo classInfo = classResolver.getClassInfo(o.getClass(), classInfoWriteCache); + return classInfo.getSerializer(); } - mapChunkWriter.writeHeader(buffer); + return null; } private void javaChunkWriteWithKVSerializers( - Fury fury, - MemoryBuffer buffer, - Map map, - Serializer keySerializer, - Serializer valueSerializer) { - MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); + MemoryBuffer buffer, Map map, Serializer keySerializer, Serializer valueSerializer) { + boolean prevKeyIsNull = false; + int header = 0; + int chunkSize = 0; + int startOffset = 0; + boolean hasPreservedByte = false; for (Object object : map.entrySet()) { Map.Entry entry = (Map.Entry) object; Object key = entry.getKey(); Object value = entry.getValue(); - mapChunkWriter = mapChunkWriter.finalKVNext(entry.getKey(), entry.getValue(), buffer); - mapChunkWriter.writeFinalKey(key, buffer, keySerializer); - mapChunkWriter.writeFinalValue(value, buffer, valueSerializer); - mapChunkWriter.increaseChunkSize(); + if (key == null) { + prevKeyIsNull = true; + } + // situations which require reset + // 1. key is null and not the first element of the chunk + // 2. prevKeyIsNull is null, we use one chunk to save data which key is null + // 3. value is null and the value is first null of values in this chunk + // 4. chunk size reach max chunk size + if ((key == null && chunkSize > 0) + || (prevKeyIsNull && key != null) + || (value == null && chunkSize > 0 && !valueHasNull(header)) + || (chunkSize >= MAX_CHUNK_SIZE)) { + writeHeader(buffer, chunkSize, header, startOffset); + header = 0; + chunkSize = 0; + hasPreservedByte = false; + prevKeyIsNull = false; + } + if (!hasPreservedByte) { + int writerIndex = buffer.writerIndex(); + // preserve two byte for header and chunk size + buffer.writerIndex(writerIndex + 2); + startOffset = writerIndex; + hasPreservedByte = true; + } + boolean trackingKeyRef = keySerializer.needToWriteRef(); + boolean trackingValueRef = valueSerializer.needToWriteRef(); + header = updateKVHeader(key, trackingKeyRef, value, trackingValueRef, header, false, false); + writeFinalKey(key, buffer, keySerializer, trackingKeyRef); + writeFinalValue(value, buffer, valueSerializer, trackingValueRef, header); + chunkSize++; + } + writeHeader(buffer, chunkSize, header, startOffset); + } + + private void writeFinalKey( + Object key, MemoryBuffer buffer, Serializer keySerializer, boolean trackingKeyRef) { + if (!trackingKeyRef) { + // map key has one null at most, use one chunk to write + if (key == null) { + buffer.writeByte(Fury.NULL_FLAG); + } else { + keySerializer.write(buffer, key); + } + } else { + RefResolver refResolver = fury.getRefResolver(); + if (!refResolver.writeRefOrNull(buffer, key)) { + keySerializer.write(buffer, key); + } } - mapChunkWriter.writeHeader(buffer); + } + + private void writeFinalValue( + Object value, + MemoryBuffer buffer, + Serializer valueSerializer, + boolean trackingValueRef, + int header) { + if (!trackingValueRef) { + if (value == null) { + buffer.writeByte(Fury.NULL_FLAG); + } else { + if (valueHasNull(header)) { + buffer.writeByte(Fury.NOT_NULL_VALUE_FLAG); + valueSerializer.write(buffer, value); + } else { + valueSerializer.write(buffer, value); + } + } + } else { + RefResolver refResolver = fury.getRefResolver(); + if (!refResolver.writeRefOrNull(buffer, value)) { + valueSerializer.write(buffer, value); + } + } + } + + private int updateKVHeader( + Object key, + boolean trackingKeyRef, + Object value, + boolean trackingValueRef, + int header, + boolean keyIsNotSameType, + boolean valueIsNotSameType) { + if (trackingKeyRef) { + header |= MapFlags.TRACKING_KEY_REF; + } + if (key == null) { + header |= MapFlags.KEY_HAS_NULL; + } + if (trackingValueRef) { + header |= MapFlags.TRACKING_VALUE_REF; + } + if (value == null) { + header |= MapFlags.VALUE_HAS_NULL; + } + if (keyIsNotSameType) { + header |= MapFlags.KEY_NOT_SAME_TYPE; + } + if (valueIsNotSameType) { + header |= MapFlags.VALUE_NOT_SAME_TYPE; + } + return header; } private void javaWriteWithKVSerializers( @@ -364,23 +725,49 @@ private void javaKVTypesFinalChunkWrite( GenericType keyGenericType, GenericType valueGenericType, Generics generics) { - MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); + boolean prevKeyIsNull = false; + int header = 0; + int chunkSize = 0; + int startOffset = 0; + boolean hasPreservedByte = false; Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); for (Object object : map.entrySet()) { Map.Entry entry = (Map.Entry) object; Object key = entry.getKey(); Object value = entry.getValue(); - mapChunkWriter = mapChunkWriter.finalKVNext(key, value, buffer); + if (key == null) { + prevKeyIsNull = true; + } + if ((key == null && chunkSize > 0) + || (prevKeyIsNull && key != null) + || (value == null && chunkSize > 0 && !valueHasNull(header)) + || (chunkSize >= MAX_CHUNK_SIZE)) { + writeHeader(buffer, chunkSize, header, startOffset); + header = 0; + chunkSize = 0; + hasPreservedByte = false; + prevKeyIsNull = false; + } + if (!hasPreservedByte) { + int writerIndex = buffer.writerIndex(); + // preserve two byte for header and chunk size + buffer.writerIndex(writerIndex + 2); + startOffset = writerIndex; + hasPreservedByte = true; + } + boolean trackingKeyRef = keySerializer.needToWriteRef(); + boolean trackingValueRef = valueSerializer.needToWriteRef(); + header = updateKVHeader(key, trackingKeyRef, value, trackingValueRef, header, false, false); generics.pushGenericType(keyGenericType); - mapChunkWriter.writeFinalKey(key, buffer, keySerializer); + writeFinalKey(key, buffer, keySerializer, trackingKeyRef); generics.popGenericType(); generics.pushGenericType(valueGenericType); - mapChunkWriter.writeFinalValue(value, buffer, valueSerializer); + writeFinalValue(value, buffer, valueSerializer, trackingValueRef, header); generics.popGenericType(); - mapChunkWriter.increaseChunkSize(); + chunkSize++; } - mapChunkWriter.writeHeader(buffer); + writeHeader(buffer, chunkSize, header, startOffset); } private void javaKeyTypeFinalWrite( @@ -419,26 +806,103 @@ private void javaKeyTypeFinalChunkWrite( GenericType keyGenericType, GenericType valueGenericType, Generics generics) { - MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); ClassResolver classResolver = fury.getClassResolver(); RefResolver refResolver = fury.getRefResolver(); Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); boolean trackingValueRef = fury.getClassResolver().needToWriteRef(valueGenericType.getCls()); + boolean prevKeyIsNull = false; + int header = 0; + int chunkSize = 0; + int startOffset = 0; + boolean hasPreservedByte = false; + boolean valueIsNotSameType = false; + Serializer valueWriteSerializer = null; + boolean writeValueClassInfo = false; + Class valueClass = null; + boolean needReset = false; for (Object object : map.entrySet()) { Map.Entry entry = (Map.Entry) object; Object key = entry.getKey(); Object value = entry.getValue(); - mapChunkWriter = mapChunkWriter.finalKeyNext(key, value, buffer); + if (key == null) { + prevKeyIsNull = true; + if (chunkSize > 0) { + needReset = true; + } + } + if (!valueIsNotSameType) { + if (value != null) { + if (valueClass == null) { + valueClass = value.getClass(); + } + valueIsNotSameType = valueClass != value.getClass(); + } + if (valueIsNotSameType) { + needReset = true; + } + } + if ((key == null && chunkSize > 0) + || (prevKeyIsNull && key != null) + || (value == null && chunkSize > 0 && !valueHasNull(header)) + || needReset + || (chunkSize >= MAX_CHUNK_SIZE)) { + writeHeader(buffer, chunkSize, header, startOffset); + prevKeyIsNull = false; + header = 0; + chunkSize = 0; + startOffset = 0; + hasPreservedByte = false; + valueWriteSerializer = null; + writeValueClassInfo = false; + valueClass = value == null ? null : value.getClass(); + needReset = false; + } + if (!hasPreservedByte) { + int writerIndex = buffer.writerIndex(); + // preserve two byte for header and chunk size + buffer.writerIndex(writerIndex + 2); + startOffset = writerIndex; + hasPreservedByte = true; + } + if (!valueIsNotSameType && valueWriteSerializer == null && !writeValueClassInfo) { + valueWriteSerializer = getSerializer(value, valueClassInfoWriteCache); + } generics.pushGenericType(keyGenericType); - mapChunkWriter.writeFinalKey(key, buffer, keySerializer); + boolean trackingKeyRef = keySerializer.needToWriteRef(); + header = + updateKVHeader( + key, trackingKeyRef, value, trackingValueRef, header, false, valueIsNotSameType); + writeFinalKey(key, buffer, keySerializer, trackingKeyRef); generics.popGenericType(); generics.pushGenericType(valueGenericType); - mapChunkWriter.writeValue( - value, buffer, classResolver, refResolver, trackingValueRef, valueClassInfoWriteCache); + if (!writeValueClassInfo && value != null) { + writeValue( + value, + buffer, + valueWriteSerializer, + classResolver, + refResolver, + trackingValueRef, + valueIsNotSameType, + header, + true); + writeValueClassInfo = true; + } else { + writeValue( + value, + buffer, + valueWriteSerializer, + classResolver, + refResolver, + trackingValueRef, + valueIsNotSameType, + header, + false); + } generics.popGenericType(); - mapChunkWriter.increaseChunkSize(); + chunkSize++; } - mapChunkWriter.writeHeader(buffer); + writeHeader(buffer, chunkSize, header, startOffset); } private void javaValueTypeFinalChunkWrite( @@ -448,26 +912,102 @@ private void javaValueTypeFinalChunkWrite( GenericType keyGenericType, GenericType valueGenericType, Generics generics) { - MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); ClassResolver classResolver = fury.getClassResolver(); RefResolver refResolver = fury.getRefResolver(); + int header = 0; + int chunkSize = 0; + boolean prevKeyIsNull = false; + boolean keyIsNotSameType = false; + int startOffset = 0; + Class keyClass = null; + Serializer keyWriteSerializer = null; + boolean writeKeyClassInfo = false; + boolean needReset = false; + boolean hasPreservedByte = false; Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); boolean trackingKeyRef = fury.getClassResolver().needToWriteRef(keyGenericType.getCls()); + boolean trackingValueRef = valueSerializer.needToWriteRef(); for (Object object : map.entrySet()) { Map.Entry entry = (Map.Entry) object; Object key = entry.getKey(); Object value = entry.getValue(); - mapChunkWriter = mapChunkWriter.finalValueNext(key, value, buffer); + if (key == null) { + prevKeyIsNull = true; + if (chunkSize > 0) { + needReset = true; + } + } + if (!keyIsNotSameType) { + if (key != null) { + if (keyClass == null) { + keyClass = key.getClass(); + } + keyIsNotSameType = keyClass != key.getClass(); + if (keyIsNotSameType) { + needReset = true; + } + } + } + if ((key == null && chunkSize > 0) + || (prevKeyIsNull && key != null) + || (value == null && chunkSize > 0 && !valueHasNull(header)) + || needReset + || (chunkSize >= MAX_CHUNK_SIZE)) { + writeHeader(buffer, chunkSize, header, startOffset); + header = 0; + chunkSize = 0; + prevKeyIsNull = false; + startOffset = 0; + keyClass = key == null ? null : key.getClass(); + keyWriteSerializer = null; + writeKeyClassInfo = false; + needReset = false; + hasPreservedByte = false; + } + header = + updateKVHeader( + key, trackingKeyRef, value, trackingValueRef, header, false, keyIsNotSameType); + if (!keyIsNotSameType && keyWriteSerializer == null && !writeKeyClassInfo) { + keyWriteSerializer = getSerializer(key, keyClassInfoWriteCache); + } + if (!hasPreservedByte) { + int writerIndex = buffer.writerIndex(); + // preserve two byte for header and chunk size + buffer.writerIndex(writerIndex + 2); + startOffset = writerIndex; + hasPreservedByte = true; + } generics.pushGenericType(keyGenericType); - mapChunkWriter.writeKey( - key, buffer, classResolver, refResolver, trackingKeyRef, keyClassInfoWriteCache); + // todo hening key == null提到外面？ + if (!writeKeyClassInfo && key != null) { + writeKey( + key, + buffer, + keyWriteSerializer, + classResolver, + refResolver, + trackingKeyRef, + keyIsNotSameType, + true); + writeKeyClassInfo = true; + } else { + writeKey( + key, + buffer, + keyWriteSerializer, + classResolver, + refResolver, + trackingKeyRef, + keyIsNotSameType, + false); + } generics.popGenericType(); generics.pushGenericType(valueGenericType); - mapChunkWriter.writeFinalValue(value, buffer, valueSerializer); + writeFinalValue(value, buffer, valueSerializer, trackingValueRef, header); generics.popGenericType(); - mapChunkWriter.increaseChunkSize(); + chunkSize++; } - mapChunkWriter.writeHeader(buffer); + writeHeader(buffer, chunkSize, header, startOffset); } private void javaKVTypesNonFinalChunkWrite( @@ -477,19 +1017,99 @@ private void javaKVTypesNonFinalChunkWrite( GenericType keyGenericType, GenericType valueGenericType, Generics generics) { - MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); ClassResolver classResolver = fury.getClassResolver(); RefResolver refResolver = fury.getRefResolver(); + int header = 0; + int startOffset = 0; + int chunkSize = 0; + Class keyClass = null; + Class valueClass = null; + boolean writeKeyClassInfo = false; + boolean writeValueClassInfo = false; + boolean keyIsNotSameType = false; + boolean valueIsNotSameType = false; + boolean prevKeyIsNull = false; + Serializer keyWriteSerializer = null; + Serializer valueWriteSerializer = null; + boolean markChunkWriteFinish = false; + boolean hasPreservedByte = false; + boolean needReset = false; + boolean needMarkFinish = false; boolean trackingKeyRef = fury.getClassResolver().needToWriteRef(keyGenericType.getCls()); boolean trackingValueRef = fury.getClassResolver().needToWriteRef(valueGenericType.getCls()); for (Object object : map.entrySet()) { Map.Entry entry = (Map.Entry) object; Object key = entry.getKey(); Object value = entry.getValue(); - mapChunkWriter = mapChunkWriter.next(key, value, buffer); - if (mapChunkWriter.isMarkChunkWriteFinish()) { + if (!markChunkWriteFinish) { + if (key == null) { + prevKeyIsNull = true; + if (chunkSize > 0) { + needReset = true; + } + } + if (!keyIsNotSameType) { + if (key != null) { + if (keyClass == null) { + keyClass = key.getClass(); + } + keyIsNotSameType = keyClass != key.getClass(); + } + if (keyIsNotSameType) { + if (valueIsNotSameType) { + needMarkFinish = true; + } else { + needReset = true; + } + } + } + if (!valueIsNotSameType) { + if (value != null) { + if (valueClass == null) { + valueClass = value.getClass(); + } + valueIsNotSameType = valueClass != value.getClass(); + } + if (valueIsNotSameType) { + if (keyIsNotSameType) { + needMarkFinish = true; + } else { + needReset = true; + } + } + } + if (needMarkFinish) { + writeHeader(buffer, chunkSize, header, startOffset); + // set chunk size = 0 + buffer.writeByte(0); + markChunkWriteFinish = true; + } else { + if ((key == null && chunkSize > 0) + || (prevKeyIsNull && key != null) + || (value == null && chunkSize > 0 && !valueHasNull(header)) + || needReset + || (chunkSize >= MAX_CHUNK_SIZE)) { + writeHeader(buffer, chunkSize, header, startOffset); + header = 0; + chunkSize = 0; + hasPreservedByte = false; + writeKeyClassInfo = false; + writeValueClassInfo = false; + prevKeyIsNull = false; + keyClass = key == null ? null : key.getClass(); + valueClass = value == null ? null : value.getClass(); + keyWriteSerializer = null; + valueWriteSerializer = null; + needReset = false; + } + } + } + if (markChunkWriteFinish) { + generics.pushGenericType(keyGenericType); writeJavaRefOptimized( fury, classResolver, refResolver, trackingKeyRef, buffer, key, keyClassInfoWriteCache); + generics.popGenericType(); + generics.pushGenericType(valueGenericType); writeJavaRefOptimized( fury, classResolver, @@ -498,19 +1118,86 @@ private void javaKVTypesNonFinalChunkWrite( buffer, value, keyClassInfoWriteCache); + generics.popGenericType(); } else { + if (!hasPreservedByte) { + int writerIndex = buffer.writerIndex(); + // preserve two byte for header and chunk size + buffer.writerIndex(writerIndex + 2); + startOffset = writerIndex; + hasPreservedByte = true; + } + + if (!keyIsNotSameType && keyWriteSerializer == null && !writeKeyClassInfo) { + keyWriteSerializer = getSerializer(key, keyClassInfoWriteCache); + } + if (!valueIsNotSameType && valueWriteSerializer == null && !writeValueClassInfo) { + valueWriteSerializer = getSerializer(value, valueClassInfoWriteCache); + } + header = + updateKVHeader( + key, + trackingKeyRef, + value, + trackingValueRef, + header, + keyIsNotSameType, + valueIsNotSameType); + boolean trackingRef = fury.trackingRef(); generics.pushGenericType(keyGenericType); - mapChunkWriter.writeKey( - key, buffer, classResolver, refResolver, trackingKeyRef, keyClassInfoWriteCache); + if (!writeKeyClassInfo && key != null) { + writeKey( + key, + buffer, + keyWriteSerializer, + classResolver, + refResolver, + trackingRef, + keyIsNotSameType, + true); + writeKeyClassInfo = true; + } else { + writeKey( + key, + buffer, + keyWriteSerializer, + classResolver, + refResolver, + trackingRef, + keyIsNotSameType, + false); + } generics.popGenericType(); generics.pushGenericType(valueGenericType); - mapChunkWriter.writeValue( - value, buffer, classResolver, refResolver, trackingValueRef, valueClassInfoWriteCache); + if (!writeValueClassInfo && value != null) { + writeValue( + value, + buffer, + valueWriteSerializer, + classResolver, + refResolver, + trackingRef, + valueIsNotSameType, + header, + true); + writeValueClassInfo = true; + } else { + writeValue( + value, + buffer, + valueWriteSerializer, + classResolver, + refResolver, + trackingRef, + valueIsNotSameType, + header, + false); + } generics.popGenericType(); - mapChunkWriter.increaseChunkSize(); + chunkSize++; } } - mapChunkWriter.writeHeader(buffer); + writeHeader(buffer, chunkSize, header, startOffset); } private void javaValueTypeFinalWrite( @@ -591,23 +1278,156 @@ private void generalJavaWrite(Fury fury, MemoryBuffer buffer, Map map) { } protected void generalJavaChunkWrite(Fury fury, MemoryBuffer buffer, Map map) { - MapChunkWriter chunkWriter = new MapChunkWriter(fury); + int header = 0; + int startOffset = 0; + int chunkSize = 0; + Class keyClass = null; + Class valueClass = null; + boolean writeKeyClassInfo = false; + boolean writeValueClassInfo = false; + boolean keyIsNotSameType = false; + boolean valueIsNotSameType = false; + boolean prevKeyIsNull = false; + Serializer keyWriteSerializer = null; + Serializer valueWriteSerializer = null; + boolean markChunkWriteFinish = false; + boolean hasPreservedByte = false; + boolean needReset = false; + boolean needMarkFinish = false; ClassResolver classResolver = fury.getClassResolver(); RefResolver refResolver = fury.getRefResolver(); for (Object object : map.entrySet()) { Map.Entry entry = (Map.Entry) object; Object key = entry.getKey(); Object value = entry.getValue(); - chunkWriter = chunkWriter.next(key, value, buffer); - if (!chunkWriter.isMarkChunkWriteFinish()) { - chunkWriter.generalChunkWrite( - key, - value, - buffer, - classResolver, - refResolver, - keyClassInfoWriteCache, - valueClassInfoWriteCache); + if (!markChunkWriteFinish) { + if (key == null) { + prevKeyIsNull = true; + if (chunkSize > 0) { + needReset = true; + } + } + if (!keyIsNotSameType) { + if (key != null) { + if (keyClass == null) { + keyClass = key.getClass(); + } + keyIsNotSameType = keyClass != key.getClass(); + } + if (keyIsNotSameType) { + if (valueIsNotSameType) { + needMarkFinish = true; + } else { + needReset = true; + } + } + } + if (!valueIsNotSameType) { + if (value != null) { + if (valueClass == null) { + valueClass = value.getClass(); + } + valueIsNotSameType = valueClass != value.getClass(); + } + if (valueIsNotSameType) { + if (keyIsNotSameType) { + needMarkFinish = true; + } else { + needReset = true; + } + } + } + if (needMarkFinish) { + writeHeader(buffer, chunkSize, header, startOffset); + // set chunk size = 0 + buffer.writeByte(0); + markChunkWriteFinish = true; + } else { + if ((key == null && chunkSize > 0) + || (prevKeyIsNull && key != null) + || (value == null && chunkSize > 0 && !valueHasNull(header)) + || needReset + || (chunkSize >= MAX_CHUNK_SIZE)) { + writeHeader(buffer, chunkSize, header, startOffset); + header = 0; + chunkSize = 0; + hasPreservedByte = false; + writeKeyClassInfo = false; + writeValueClassInfo = false; + prevKeyIsNull = false; + keyClass = key == null ? null : key.getClass(); + valueClass = value == null ? null : value.getClass(); + keyWriteSerializer = null; + valueWriteSerializer = null; + needReset = false; + } + } + } + if (!markChunkWriteFinish) { + if (!hasPreservedByte) { + int writerIndex = buffer.writerIndex(); + // preserve two byte for header and chunk size + buffer.writerIndex(writerIndex + 2); + startOffset = writerIndex; + hasPreservedByte = true; + } + if (!keyIsNotSameType && keyWriteSerializer == null && !writeKeyClassInfo) { + keyWriteSerializer = getSerializer(key, keyClassInfoWriteCache); + } + if (!valueIsNotSameType && valueWriteSerializer == null && !writeValueClassInfo) { + valueWriteSerializer = getSerializer(value, valueClassInfoWriteCache); + } + boolean trackingRef = fury.trackingRef(); + header = + updateKVHeader( + key, trackingRef, value, trackingRef, header, keyIsNotSameType, valueIsNotSameType); + if (!writeKeyClassInfo && key != null) { + writeKey( + key, + buffer, + keyWriteSerializer, + classResolver, + refResolver, + trackingRef, + keyIsNotSameType, + true); + writeKeyClassInfo = true; + } else { + writeKey( + key, + buffer, + keyWriteSerializer, + classResolver, + refResolver, + trackingRef, + keyIsNotSameType, + false); + } + if (!writeValueClassInfo && value != null) { + writeValue( + value, + buffer, + valueWriteSerializer, + classResolver, + refResolver, + trackingRef, + valueIsNotSameType, + header, + true); + writeValueClassInfo = true; + } else { + writeValue( + value, + buffer, + valueWriteSerializer, + classResolver, + refResolver, + trackingRef, + valueIsNotSameType, + header, + false); + } + chunkSize++; } else { writeJavaRefOptimized( fury, classResolver, refResolver, buffer, entry.getKey(), keyClassInfoWriteCache); @@ -615,7 +1435,32 @@ protected void generalJavaChunkWrite(Fury fury, MemoryBuffer buffer, Map map) { fury, classResolver, refResolver, buffer, entry.getValue(), valueClassInfoWriteCache); } } - chunkWriter.writeHeader(buffer); + writeHeader(buffer, chunkSize, header, startOffset); + } + + private void writeNoNullRef( + Serializer serializer, Object o, MemoryBuffer buffer, RefResolver refResolver) { + if (serializer.needToWriteRef()) { + if (!refResolver.writeRefOrNull(buffer, o)) { + serializer.write(buffer, o); + } + } else { + serializer.write(buffer, o); + } + } + + private boolean valueHasNull(int header) { + return (header & MapFlags.VALUE_HAS_NULL) == MapFlags.VALUE_HAS_NULL; + } + + public void writeHeader(MemoryBuffer memoryBuffer, int chunkSize, int header, int startOffset) { + if (chunkSize > 0) { + int currentWriteIndex = memoryBuffer.writerIndex(); + memoryBuffer.writerIndex(startOffset); + memoryBuffer.writeByte(chunkSize); + memoryBuffer.writeByte(header); + memoryBuffer.writerIndex(currentWriteIndex); + } } public static void xwriteElements(Fury fury, MemoryBuffer buffer, Map value) { @@ -781,24 +1626,66 @@ private void genericJavaRead(Fury fury, MemoryBuffer buffer, Map map, int size) private void javaChunkReadWithKeySerializer( MemoryBuffer buffer, Map map, int size, Serializer keySerializer) { - MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); + final ClassResolver classResolver = fury.getClassResolver(); while (size > 0) { byte chunkSize = buffer.readByte(); byte header = buffer.readByte(); Preconditions.checkArgument( chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); + Serializer valueReadSerializer = null; for (byte i = 0; i < chunkSize; i++) { Object key; Object value; - key = mapChunkWriter.readFinalKey(buffer, header, keySerializer); - value = - mapChunkWriter.readValue( - header, - buffer, - fury.getClassResolver(), - fury.trackingRef(), - valueClassInfoReadCache); + key = readFinalKey(buffer, header, keySerializer); + if (!fury.trackingRef()) { + if (!valueIsNotSameType(header)) { + if (valueHasNull(header)) { + byte flag = buffer.readByte(); + if (flag == Fury.NOT_NULL_VALUE_FLAG) { + if (valueReadSerializer == null) { + valueReadSerializer = + classResolver.readClassInfo(buffer, valueClassInfoReadCache).getSerializer(); + } + value = valueReadSerializer.read(buffer); + } else { + value = null; + } + } else { + if (valueReadSerializer == null) { + valueReadSerializer = + classResolver.readClassInfo(buffer, valueClassInfoReadCache).getSerializer(); + } + value = valueReadSerializer.read(buffer); + } + } else { + value = fury.readNullable(buffer, valueClassInfoReadCache); + } + + } else { + if (!valueIsNotSameType(header)) { + if (valueHasNull(header)) { + byte flag = buffer.readByte(); + if (flag == Fury.NOT_NULL_VALUE_FLAG) { + if (valueReadSerializer == null) { + valueReadSerializer = + classResolver.readClassInfo(buffer, valueClassInfoReadCache).getSerializer(); + } + value = fury.readRef(buffer, valueReadSerializer); + } else { + value = null; + } + } else { + if (valueReadSerializer == null) { + valueReadSerializer = + classResolver.readClassInfo(buffer, valueClassInfoReadCache).getSerializer(); + } + value = readNoNullRef(valueReadSerializer, buffer); + } + } else { + value = fury.readRef(buffer, valueClassInfoReadCache); + } + } map.put(key, value); size--; } @@ -807,20 +1694,54 @@ private void javaChunkReadWithKeySerializer( private void javaChunkReadWithValueSerializer( MemoryBuffer buffer, Map map, int size, Serializer valueSerializer) { - MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); while (size > 0) { byte chunkSize = buffer.readByte(); byte header = buffer.readByte(); Preconditions.checkArgument( chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); + Serializer keyReadSerializer = null; for (byte i = 0; i < chunkSize; i++) { Object key; Object value; - key = - mapChunkWriter.readKey( - header, buffer, fury.getClassResolver(), fury.trackingRef(), keyClassInfoReadCache); - value = mapChunkWriter.readFinalValue(buffer, header, valueSerializer); + if (!fury.trackingRef()) { + if (keyHasNull(header)) { + byte nullFlag = buffer.readByte(); + Preconditions.checkArgument(nullFlag == Fury.NULL_FLAG, "unexpected error"); + key = null; + } else { + if (!keyIsNotSameType(header)) { + if (keyReadSerializer == null) { + keyReadSerializer = + fury.getClassResolver() + .readClassInfo(buffer, keyClassInfoReadCache) + .getSerializer(); + } + key = keyReadSerializer.read(buffer); + } else { + key = fury.readNonRef(buffer, keyClassInfoReadCache); + } + } + } else { + if (keyHasNull(header)) { + byte nullFlag = buffer.readByte(); + Preconditions.checkArgument(nullFlag == Fury.NULL_FLAG, "unexpected error"); + key = null; + } else { + if (!keyIsNotSameType(header)) { + if (keyReadSerializer == null) { + keyReadSerializer = + fury.getClassResolver() + .readClassInfo(buffer, keyClassInfoReadCache) + .getSerializer(); + } + key = readNoNullRef(keyReadSerializer, buffer); + } else { + key = fury.readRef(buffer, keyClassInfoReadCache); + } + } + } + value = readFinalValue(buffer, header, valueSerializer); map.put(key, value); size--; } @@ -833,7 +1754,6 @@ private void javaChunkReadWithKVSerializers( int size, Serializer keySerializer, Serializer valueSerializer) { - MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); while (size > 0) { byte chunkSize = buffer.readByte(); byte header = buffer.readByte(); @@ -843,14 +1763,47 @@ private void javaChunkReadWithKVSerializers( for (byte i = 0; i < chunkSize; i++) { Object key; Object value; - key = mapChunkWriter.readFinalKey(buffer, header, keySerializer); - value = mapChunkWriter.readFinalValue(buffer, header, valueSerializer); + key = readFinalKey(buffer, header, keySerializer); + value = readFinalValue(buffer, header, valueSerializer); map.put(key, value); size--; } } } + public Object readFinalKey(MemoryBuffer buffer, int header, Serializer keySerializer) { + boolean trackingKeyRef = keySerializer.needToWriteRef(); + if (!trackingKeyRef) { + if (keyHasNull(header)) { + byte nullFlag = buffer.readByte(); + Preconditions.checkArgument(nullFlag == Fury.NULL_FLAG, "unexpected NULL_FLAG"); + return null; + } else { + return keySerializer.read(buffer); + } + } else { + return fury.readRef(buffer, keySerializer); + } + } + + public Object readFinalValue(MemoryBuffer buffer, int header, Serializer valueSerializer) { + boolean trackingValueRef = valueSerializer.needToWriteRef(); + if (!trackingValueRef) { + if (valueHasNull(header)) { + byte flag = buffer.readByte(); + if (flag == Fury.NOT_NULL_VALUE_FLAG) { + return valueSerializer.read(buffer); + } else { + return null; + } + } else { + return valueSerializer.read(buffer); + } + } else { + return fury.readRef(buffer, valueSerializer); + } + } + private void genericJavaChunkRead(Fury fury, MemoryBuffer buffer, Map map, int size) { Generics generics = fury.getGenerics(); GenericType genericType = generics.nextGenericType(); @@ -901,8 +1854,6 @@ private void javaKVTypesFinalChunkRead( while (size > 0) { byte chunkSize = buffer.readByte(); byte header = buffer.readByte(); - mapChunkWriter.setKeyReadSerializer(null); - mapChunkWriter.setValueReadSerializer(null); Preconditions.checkArgument( chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); @@ -910,10 +1861,10 @@ private void javaKVTypesFinalChunkRead( Object key; Object value; generics.pushGenericType(keyGenericType); - key = mapChunkWriter.readFinalKey(buffer, header, keySerializer); + key = readFinalKey(buffer, header, keySerializer); generics.popGenericType(); generics.pushGenericType(valueGenericType); - value = mapChunkWriter.readFinalValue(buffer, header, valueSerializer); + value = readFinalValue(buffer, header, valueSerializer); generics.popGenericType(); map.put(key, value); size--; @@ -950,7 +1901,6 @@ private void javaKeyTypeFinalChunkRead( GenericType valueGenericType, Generics generics, int size) { - final MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); ClassResolver classResolver = fury.getClassResolver(); boolean trackingValueRef = classResolver.needToWriteRef(valueGenericType.getCls()); Serializer keySerializer = keyGenericType.getSerializer(fury.getClassResolver()); @@ -960,16 +1910,61 @@ private void javaKeyTypeFinalChunkRead( chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); byte header = buffer.readByte(); - mapChunkWriter.setKeyReadSerializer(null); - mapChunkWriter.setValueReadSerializer(null); + Serializer valueReadSerializer = null; while (chunkSize > 0) { generics.pushGenericType(keyGenericType); - Object key = mapChunkWriter.readFinalKey(buffer, header, keySerializer); - generics.pushGenericType(valueGenericType); - Object value = - mapChunkWriter.readValue( - header, buffer, classResolver, trackingValueRef, valueClassInfoReadCache); + Object key = readFinalKey(buffer, header, keySerializer); + generics.popGenericType(); generics.pushGenericType(valueGenericType); + Object value; + if (!trackingValueRef) { + if (!valueIsNotSameType(header)) { + if (valueHasNull(header)) { + byte flag = buffer.readByte(); + if (flag == Fury.NOT_NULL_VALUE_FLAG) { + if (valueReadSerializer == null) { + valueReadSerializer = + classResolver.readClassInfo(buffer, valueClassInfoReadCache).getSerializer(); + } + value = valueReadSerializer.read(buffer); + } else { + value = null; + } + } else { + if (valueReadSerializer == null) { + valueReadSerializer = + classResolver.readClassInfo(buffer, valueClassInfoReadCache).getSerializer(); + } + value = valueReadSerializer.read(buffer); + } + } else { + value = fury.readNullable(buffer, valueClassInfoReadCache); + } + + } else { + if (!valueIsNotSameType(header)) { + if (valueHasNull(header)) { + byte flag = buffer.readByte(); + if (flag == Fury.NOT_NULL_VALUE_FLAG) { + if (valueReadSerializer == null) { + valueReadSerializer = + classResolver.readClassInfo(buffer, valueClassInfoReadCache).getSerializer(); + } + value = fury.readRef(buffer, valueReadSerializer); + } else { + value = null; + } + } else { + if (valueReadSerializer == null) { + valueReadSerializer = + classResolver.readClassInfo(buffer, valueClassInfoReadCache).getSerializer(); + } + value = readNoNullRef(valueReadSerializer, buffer); + } + } else { + value = fury.readRef(buffer, valueClassInfoReadCache); + } + } generics.popGenericType(); chunkSize--; size--; @@ -1010,26 +2005,58 @@ private void javaValueTypeFinalChunkRead( GenericType valueGenericType, Generics generics, int size) { - MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); boolean trackingKeyRef = fury.getClassResolver().needToWriteRef(keyGenericType.getCls()); Serializer valueSerializer = valueGenericType.getSerializer(fury.getClassResolver()); - ClassResolver classResolver = fury.getClassResolver(); while (size > 0) { byte chunkSize = buffer.readByte(); Preconditions.checkArgument( chunkSize >= 0, "chunkSize < 0, which means serialization protocol is not same with deserialization protocol"); byte header = buffer.readByte(); - mapChunkWriter.setKeyReadSerializer(null); - mapChunkWriter.setValueReadSerializer(null); + Serializer keyReadSerializer = null; while (chunkSize > 0) { generics.pushGenericType(keyGenericType); - Object key = - mapChunkWriter.readKey( - header, buffer, classResolver, trackingKeyRef, keyClassInfoReadCache); - generics.pushGenericType(valueGenericType); - Object value = mapChunkWriter.readFinalValue(buffer, header, valueSerializer); + Object key; + if (!trackingKeyRef) { + if (keyHasNull(header)) { + byte nullFlag = buffer.readByte(); + Preconditions.checkArgument(nullFlag == Fury.NULL_FLAG, "unexpected error"); + key = null; + } else { + if (!keyIsNotSameType(header)) { + if (keyReadSerializer == null) { + keyReadSerializer = + fury.getClassResolver() + .readClassInfo(buffer, keyClassInfoReadCache) + .getSerializer(); + } + key = keyReadSerializer.read(buffer); + } else { + key = fury.readNonRef(buffer, keyClassInfoReadCache); + } + } + } else { + if (keyHasNull(header)) { + byte nullFlag = buffer.readByte(); + Preconditions.checkArgument(nullFlag == Fury.NULL_FLAG, "unexpected error"); + key = null; + } else { + if (!keyIsNotSameType(header)) { + if (keyReadSerializer == null) { + keyReadSerializer = + fury.getClassResolver() + .readClassInfo(buffer, keyClassInfoReadCache) + .getSerializer(); + } + key = readNoNullRef(keyReadSerializer, buffer); + } else { + key = fury.readRef(buffer, keyClassInfoReadCache); + } + } + } + generics.popGenericType(); generics.pushGenericType(valueGenericType); + Object value = readFinalValue(buffer, header, valueSerializer); generics.popGenericType(); chunkSize--; size--; @@ -1069,7 +2096,6 @@ private void javaKVTypesNonFinalChunkRead( GenericType valueGenericType, Generics generics, int size) { - final MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); ClassResolver classResolver = fury.getClassResolver(); RefResolver refResolver = fury.getRefResolver(); boolean trackingKeyRef = classResolver.needToWriteRef(keyGenericType.getCls()); @@ -1096,18 +2122,99 @@ private void javaKVTypesNonFinalChunkRead( } } else { byte header = buffer.readByte(); - mapChunkWriter.setKeyReadSerializer(null); - mapChunkWriter.setValueReadSerializer(null); + Serializer keyReadSerializer = null; + Serializer valueReadSerializer = null; while (chunkSize > 0) { generics.pushGenericType(keyGenericType); - Object key = - mapChunkWriter.readKey( - header, buffer, classResolver, trackingKeyRef, keyClassInfoReadCache); + Object key; + if (!trackingKeyRef) { + if (keyHasNull(header)) { + byte nullFlag = buffer.readByte(); + Preconditions.checkArgument(nullFlag == Fury.NULL_FLAG, "unexpected error"); + key = null; + } else { + if (!keyIsNotSameType(header)) { + if (keyReadSerializer == null) { + keyReadSerializer = + classResolver.readClassInfo(buffer, keyClassInfoReadCache).getSerializer(); + } + key = keyReadSerializer.read(buffer); + } else { + key = fury.readNonRef(buffer, keyClassInfoReadCache); + } + } + } else { + if (keyHasNull(header)) { + byte nullFlag = buffer.readByte(); + Preconditions.checkArgument(nullFlag == Fury.NULL_FLAG, "unexpected error"); + key = null; + } else { + if (!keyIsNotSameType(header)) { + if (keyReadSerializer == null) { + keyReadSerializer = + classResolver.readClassInfo(buffer, keyClassInfoReadCache).getSerializer(); + } + key = readNoNullRef(keyReadSerializer, buffer); + } else { + key = fury.readRef(buffer, keyClassInfoReadCache); + } + } + } generics.popGenericType(); generics.pushGenericType(valueGenericType); - Object value = - mapChunkWriter.readValue( - header, buffer, classResolver, trackingValueRef, valueClassInfoReadCache); + Object value; + if (!trackingValueRef) { + if (!valueIsNotSameType(header)) { + if (valueHasNull(header)) { + byte flag = buffer.readByte(); + if (flag == Fury.NOT_NULL_VALUE_FLAG) { + if (valueReadSerializer == null) { + valueReadSerializer = + classResolver + .readClassInfo(buffer, valueClassInfoReadCache) + .getSerializer(); + } + value = valueReadSerializer.read(buffer); + } else { + value = null; + } + } else { + if (valueReadSerializer == null) { + valueReadSerializer = + classResolver.readClassInfo(buffer, valueClassInfoReadCache).getSerializer(); + } + value = valueReadSerializer.read(buffer); + } + } else { + value = fury.readNullable(buffer, valueClassInfoReadCache); + } + + } else { + if (!valueIsNotSameType(header)) { + if (valueHasNull(header)) { + byte flag = buffer.readByte(); + if (flag == Fury.NOT_NULL_VALUE_FLAG) { + if (valueReadSerializer == null) { + valueReadSerializer = + classResolver + .readClassInfo(buffer, valueClassInfoReadCache) + .getSerializer(); + } + value = fury.readRef(buffer, valueReadSerializer); + } else { + value = null; + } + } else { + if (valueReadSerializer == null) { + valueReadSerializer = + classResolver.readClassInfo(buffer, valueClassInfoReadCache).getSerializer(); + } + value = readNoNullRef(valueReadSerializer, buffer); + } + } else { + value = fury.readRef(buffer, valueClassInfoReadCache); + } + } generics.popGenericType(); chunkSize--; size--; @@ -1144,9 +2251,8 @@ private void javaKVTypesNonFinalRead( } private void generalJavaChunkRead(Fury fury, MemoryBuffer buffer, Map map, int size) { - final MapChunkWriter mapChunkWriter = new MapChunkWriter(fury); ClassResolver classResolver = fury.getClassResolver(); - boolean trackingKeyRef = fury.trackingRef(); + boolean trackingRef = fury.trackingRef(); while (size > 0) { byte chunkSize = buffer.readByte(); Preconditions.checkArgument( @@ -1161,15 +2267,96 @@ private void generalJavaChunkRead(Fury fury, MemoryBuffer buffer, Map map, int s } } else { byte header = buffer.readByte(); - mapChunkWriter.setKeyReadSerializer(null); - mapChunkWriter.setValueReadSerializer(null); + Serializer keyReadSerializer = null; + Serializer valueReadSerializer = null; while (chunkSize > 0) { - Object key = - mapChunkWriter.readKey( - header, buffer, classResolver, trackingKeyRef, keyClassInfoReadCache); - Object value = - mapChunkWriter.readValue( - header, buffer, classResolver, trackingKeyRef, valueClassInfoReadCache); + Object key; + if (!trackingRef) { + if (keyHasNull(header)) { + byte nullFlag = buffer.readByte(); + Preconditions.checkArgument(nullFlag == Fury.NULL_FLAG, "unexpected error"); + key = null; + } else { + if (!keyIsNotSameType(header)) { + if (keyReadSerializer == null) { + keyReadSerializer = + classResolver.readClassInfo(buffer, keyClassInfoReadCache).getSerializer(); + } + key = keyReadSerializer.read(buffer); + } else { + key = fury.readNonRef(buffer, keyClassInfoReadCache); + } + } + } else { + if (keyHasNull(header)) { + byte nullFlag = buffer.readByte(); + Preconditions.checkArgument(nullFlag == Fury.NULL_FLAG, "unexpected error"); + key = null; + } else { + if (!keyIsNotSameType(header)) { + if (keyReadSerializer == null) { + keyReadSerializer = + classResolver.readClassInfo(buffer, keyClassInfoReadCache).getSerializer(); + } + key = readNoNullRef(keyReadSerializer, buffer); + } else { + key = fury.readRef(buffer, keyClassInfoReadCache); + } + } + } + Object value; + if (!trackingRef) { + if (!valueIsNotSameType(header)) { + if (valueHasNull(header)) { + byte flag = buffer.readByte(); + if (flag == Fury.NOT_NULL_VALUE_FLAG) { + if (valueReadSerializer == null) { + valueReadSerializer = + classResolver + .readClassInfo(buffer, valueClassInfoReadCache) + .getSerializer(); + } + value = valueReadSerializer.read(buffer); + } else { + value = null; + } + } else { + if (valueReadSerializer == null) { + valueReadSerializer = + classResolver.readClassInfo(buffer, valueClassInfoReadCache).getSerializer(); + } + value = valueReadSerializer.read(buffer); + } + } else { + value = fury.readNullable(buffer, valueClassInfoReadCache); + } + + } else { + if (!valueIsNotSameType(header)) { + if (valueHasNull(header)) { + byte flag = buffer.readByte(); + if (flag == Fury.NOT_NULL_VALUE_FLAG) { + if (valueReadSerializer == null) { + valueReadSerializer = + classResolver + .readClassInfo(buffer, valueClassInfoReadCache) + .getSerializer(); + } + value = fury.readRef(buffer, valueReadSerializer); + } else { + value = null; + } + } else { + if (valueReadSerializer == null) { + valueReadSerializer = + classResolver.readClassInfo(buffer, valueClassInfoReadCache).getSerializer(); + } + value = readNoNullRef(valueReadSerializer, buffer); + } + } else { + value = fury.readRef(buffer, valueClassInfoReadCache); + } + } chunkSize--; size--; map.put(key, value); @@ -1178,6 +2365,34 @@ private void generalJavaChunkRead(Fury fury, MemoryBuffer buffer, Map map, int s } } + private boolean keyHasNull(int header) { + return (header & MapFlags.KEY_HAS_NULL) == MapFlags.KEY_HAS_NULL; + } + + private boolean keyIsNotSameType(int header) { + return (header & MapFlags.KEY_NOT_SAME_TYPE) == MapFlags.KEY_NOT_SAME_TYPE; + } + + private boolean valueIsNotSameType(int header) { + return (header & MapFlags.VALUE_NOT_SAME_TYPE) == MapFlags.VALUE_NOT_SAME_TYPE; + } + + private Object readNoNullRef(Serializer serializer, MemoryBuffer memoryBuffer) { + if (serializer.needToWriteRef()) { + final RefResolver refResolver = fury.getRefResolver(); + int nextReadRefId = refResolver.tryPreserveRefId(memoryBuffer); + if (nextReadRefId >= Fury.NOT_NULL_VALUE_FLAG) { + Object obj = serializer.read(memoryBuffer); + refResolver.setReadObject(nextReadRefId, obj); + return obj; + } else { + return refResolver.getReadObject(); + } + } else { + return serializer.read(memoryBuffer); + } + } + private void generalJavaRead(Fury fury, MemoryBuffer buffer, Map map, int size) { for (int i = 0; i < size; i++) { Object key = fury.readRef(buffer, keyClassInfoReadCache); From aa364f51cde5cd64b9cd007a2a611a20452588e8 Mon Sep 17 00:00:00 2001 From: hening Date: Wed, 11 Sep 2024 11:07:26 +0800 Subject: [PATCH 18/18] use local variable and big method --- .../collection/AbstractMapSerializer.java | 694 ++++++++++-------- 1 file changed, 390 insertions(+), 304 deletions(-) diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java index 216085f68a..1b175c9db1 100644 --- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java +++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java @@ -224,142 +224,66 @@ private void javaChunkWriteWithKeySerializers( // write final key boolean trackingKeyRef = keySerializer.needToWriteRef(); boolean trackingValueRef = fury.trackingRef(); - if (!valueIsNotSameType && valueWriteSerializer == null && !writeValueClassInfo) { - valueWriteSerializer = getSerializer(value, valueClassInfoWriteCache); - } header = updateKVHeader( key, trackingKeyRef, value, trackingValueRef, header, false, valueIsNotSameType); writeFinalKey(key, buffer, keySerializer, trackingKeyRef); - if (!writeValueClassInfo && valueWriteSerializer != null) { - writeValue( - value, - buffer, - valueWriteSerializer, - classResolver, - refResolver, - trackingValueRef, - valueIsNotSameType, - header, - true); - writeValueClassInfo = true; - } else { - writeValue( - value, - buffer, - valueWriteSerializer, - classResolver, - refResolver, - trackingValueRef, - valueIsNotSameType, - header, - false); - } - chunkSize++; - } - writeHeader(buffer, chunkSize, header, startOffset); - } - - private void writeKey( - Object key, - MemoryBuffer buffer, - Serializer keyWriteSerializer, - ClassResolver classResolver, - RefResolver refResolver, - boolean trackingKeyRef, - boolean keyIsNotSameType, - boolean needWriteClass) { - if (!trackingKeyRef) { - if (key == null) { - buffer.writeByte(Fury.NULL_FLAG); - } else { - if (!keyIsNotSameType) { - writeClass(key, buffer, valueClassInfoWriteCache, needWriteClass); - keyWriteSerializer.write(buffer, key); - } else { - fury.writeNonRef( - buffer, key, classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache)); - } - } - } else { - // todo 提到外面 - if (key == null) { - buffer.writeByte(Fury.NULL_FLAG); - } else { - if (!keyIsNotSameType) { - // todo key is not null, no need to write no null flag - writeClass(key, buffer, valueClassInfoWriteCache, needWriteClass); - writeNoNullRef(keyWriteSerializer, key, buffer, refResolver); - } else { - if (!refResolver.writeNullFlag(buffer, key)) { - fury.writeRef( - buffer, key, classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache)); - } - } - } - } - } - - private void writeValue( - Object value, - MemoryBuffer buffer, - Serializer valueWriteSerializer, - ClassResolver classResolver, - RefResolver refResolver, - boolean trackingValueRef, - boolean valueIsNotSameType, - int header, - boolean needWriteClass) { - if (!trackingValueRef) { - if (value == null) { - buffer.writeByte(Fury.NULL_FLAG); - } else { - if (!valueIsNotSameType) { - if (valueHasNull(header)) { - buffer.writeByte(Fury.NOT_NULL_VALUE_FLAG); - } - writeClass(value, buffer, valueClassInfoWriteCache, needWriteClass); - valueWriteSerializer.write(buffer, value); - } else { - fury.writeNullable( - buffer, - value, - classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache)); - } - } - } else { - if (value == null) { - buffer.writeByte(Fury.NULL_FLAG); - } else { - if (!valueIsNotSameType) { - writeClass(value, buffer, valueClassInfoWriteCache, needWriteClass); - if (!valueHasNull(header)) { - writeNoNullRef(valueWriteSerializer, value, buffer, refResolver); - } else { - fury.writeRef(buffer, value, valueWriteSerializer); - } + if (!trackingValueRef) { + if (value == null) { + buffer.writeByte(Fury.NULL_FLAG); + } else { + if (!valueIsNotSameType) { + if (valueHasNull(header)) { + buffer.writeByte(Fury.NOT_NULL_VALUE_FLAG); + } + ClassInfo classInfo = classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache); + if (!writeValueClassInfo) { + classResolver.writeClass(buffer, classInfo); + writeValueClassInfo = true; + } + if (valueWriteSerializer == null) { + valueWriteSerializer = classInfo.getSerializer(); + } + valueWriteSerializer.write(buffer, value); + } else { + fury.writeNullable( + buffer, + value, + classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache)); + } + } } else { - if (!refResolver.writeNullFlag(buffer, value)) { - fury.writeRef( - buffer, - value, - classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache)); - } + if (value == null) { + buffer.writeByte(Fury.NULL_FLAG); + } else { + if (!valueIsNotSameType) { + ClassInfo classInfo = classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache); + if (!writeValueClassInfo) { + classResolver.writeClass(buffer, classInfo); + writeValueClassInfo = true; + } + if (valueWriteSerializer == null) { + valueWriteSerializer = classInfo.getSerializer(); + } + valueWriteSerializer.write(buffer, value); + if (!valueHasNull(header)) { + writeNoNullRef(valueWriteSerializer, value, buffer, refResolver); + } else { + fury.writeRef(buffer, value, valueWriteSerializer); + } + } else { + if (!refResolver.writeNullFlag(buffer, value)) { + fury.writeRef( + buffer, + value, + classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache)); + } + } + } } - } - } - } - - private void writeClass( - Object o, - MemoryBuffer memoryBuffer, - ClassInfoHolder classInfoWriteCache, - boolean needWriteClass) { - ClassResolver classResolver = fury.getClassResolver(); - ClassInfo classInfo = classResolver.getClassInfo(o.getClass(), classInfoWriteCache); - if (needWriteClass) { - classResolver.writeClass(memoryBuffer, classInfo); + chunkSize++; } + writeHeader(buffer, chunkSize, header, startOffset); } private void javaChunkWriteWithValueSerializers( @@ -418,34 +342,51 @@ private void javaChunkWriteWithValueSerializers( } boolean trackingKeyRef = fury.trackingRef(); boolean trackingValueRef = valueSerializer.needToWriteRef(); - if (!keyIsNotSameType && keyWriteSerializer == null && !writeKeyClassInfo) { - keyWriteSerializer = getSerializer(key, keyClassInfoWriteCache); - } header = updateKVHeader( key, trackingKeyRef, value, trackingValueRef, header, keyIsNotSameType, false); - if (!writeKeyClassInfo && key != null) { - writeKey( - key, - buffer, - keyWriteSerializer, - classResolver, - refResolver, - trackingKeyRef, - keyIsNotSameType, - true); - writeKeyClassInfo = true; - } else { - writeKey( - key, - buffer, - keyWriteSerializer, - classResolver, - refResolver, - trackingKeyRef, - keyIsNotSameType, - false); - } + if (!trackingKeyRef) { + if (key == null) { + buffer.writeByte(Fury.NULL_FLAG); + } else { + if (!keyIsNotSameType) { + ClassInfo classInfo = classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache); + if (!writeKeyClassInfo) { + classResolver.writeClass(buffer, classInfo); + writeKeyClassInfo = true; + } + if (keyWriteSerializer == null) { + keyWriteSerializer = classInfo.getSerializer(); + } + keyWriteSerializer.write(buffer, key); + } else { + fury.writeNonRef( + buffer, key, classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache)); + } + } + } else { + // todo 提到外面 + if (key == null) { + buffer.writeByte(Fury.NULL_FLAG); + } else { + if (!keyIsNotSameType) { + ClassInfo classInfo = classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache); + if (!writeKeyClassInfo) { + classResolver.writeClass(buffer, classInfo); + writeKeyClassInfo = true; + } + if (keyWriteSerializer == null) { + keyWriteSerializer = classInfo.getSerializer(); + } + writeNoNullRef(keyWriteSerializer, key, buffer, refResolver); + } else { + if (!refResolver.writeNullFlag(buffer, key)) { + fury.writeRef( + buffer, key, classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache)); + } + } + } + } writeFinalValue(value, buffer, valueSerializer, trackingValueRef, header); chunkSize++; } @@ -864,9 +805,6 @@ private void javaKeyTypeFinalChunkWrite( startOffset = writerIndex; hasPreservedByte = true; } - if (!valueIsNotSameType && valueWriteSerializer == null && !writeValueClassInfo) { - valueWriteSerializer = getSerializer(value, valueClassInfoWriteCache); - } generics.pushGenericType(keyGenericType); boolean trackingKeyRef = keySerializer.needToWriteRef(); header = @@ -875,30 +813,59 @@ private void javaKeyTypeFinalChunkWrite( writeFinalKey(key, buffer, keySerializer, trackingKeyRef); generics.popGenericType(); generics.pushGenericType(valueGenericType); - if (!writeValueClassInfo && value != null) { - writeValue( - value, - buffer, - valueWriteSerializer, - classResolver, - refResolver, - trackingValueRef, - valueIsNotSameType, - header, - true); - writeValueClassInfo = true; - } else { - writeValue( - value, - buffer, - valueWriteSerializer, - classResolver, - refResolver, - trackingValueRef, - valueIsNotSameType, - header, - false); - } + if (!trackingValueRef) { + if (value == null) { + buffer.writeByte(Fury.NULL_FLAG); + } else { + if (!valueIsNotSameType) { + if (valueHasNull(header)) { + buffer.writeByte(Fury.NOT_NULL_VALUE_FLAG); + } + ClassInfo classInfo = classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache); + if (!writeValueClassInfo) { + classResolver.writeClass(buffer, classInfo); + writeValueClassInfo = true; + } + if (valueWriteSerializer == null) { + valueWriteSerializer = classInfo.getSerializer(); + } + valueWriteSerializer.write(buffer, value); + } else { + fury.writeNullable( + buffer, + value, + classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache)); + } + } + } else { + if (value == null) { + buffer.writeByte(Fury.NULL_FLAG); + } else { + if (!valueIsNotSameType) { + ClassInfo classInfo = classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache); + if (!writeValueClassInfo) { + classResolver.writeClass(buffer, classInfo); + writeValueClassInfo = true; + } + if (valueWriteSerializer == null) { + valueWriteSerializer = classInfo.getSerializer(); + } + valueWriteSerializer.write(buffer, value); + if (!valueHasNull(header)) { + writeNoNullRef(valueWriteSerializer, value, buffer, refResolver); + } else { + fury.writeRef(buffer, value, valueWriteSerializer); + } + } else { + if (!refResolver.writeNullFlag(buffer, value)) { + fury.writeRef( + buffer, + value, + classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache)); + } + } + } + } generics.popGenericType(); chunkSize++; } @@ -967,9 +934,6 @@ private void javaValueTypeFinalChunkWrite( header = updateKVHeader( key, trackingKeyRef, value, trackingValueRef, header, false, keyIsNotSameType); - if (!keyIsNotSameType && keyWriteSerializer == null && !writeKeyClassInfo) { - keyWriteSerializer = getSerializer(key, keyClassInfoWriteCache); - } if (!hasPreservedByte) { int writerIndex = buffer.writerIndex(); // preserve two byte for header and chunk size @@ -979,28 +943,48 @@ private void javaValueTypeFinalChunkWrite( } generics.pushGenericType(keyGenericType); // todo hening key == null提到外面？ - if (!writeKeyClassInfo && key != null) { - writeKey( - key, - buffer, - keyWriteSerializer, - classResolver, - refResolver, - trackingKeyRef, - keyIsNotSameType, - true); - writeKeyClassInfo = true; - } else { - writeKey( - key, - buffer, - keyWriteSerializer, - classResolver, - refResolver, - trackingKeyRef, - keyIsNotSameType, - false); - } + if (!trackingKeyRef) { + if (key == null) { + buffer.writeByte(Fury.NULL_FLAG); + } else { + if (!keyIsNotSameType) { + ClassInfo classInfo = classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache); + if (!writeKeyClassInfo) { + classResolver.writeClass(buffer, classInfo); + writeKeyClassInfo = true; + } + if (keyWriteSerializer == null) { + keyWriteSerializer = classInfo.getSerializer(); + } + keyWriteSerializer.write(buffer, key); + } else { + fury.writeNonRef( + buffer, key, classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache)); + } + } + } else { + // todo 提到外面 + if (key == null) { + buffer.writeByte(Fury.NULL_FLAG); + } else { + if (!keyIsNotSameType) { + ClassInfo classInfo = classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache); + if (!writeKeyClassInfo) { + classResolver.writeClass(buffer, classInfo); + writeKeyClassInfo = true; + } + if (keyWriteSerializer == null) { + keyWriteSerializer = classInfo.getSerializer(); + } + writeNoNullRef(keyWriteSerializer, key, buffer, refResolver); + } else { + if (!refResolver.writeNullFlag(buffer, key)) { + fury.writeRef( + buffer, key, classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache)); + } + } + } + } generics.popGenericType(); generics.pushGenericType(valueGenericType); writeFinalValue(value, buffer, valueSerializer, trackingValueRef, header); @@ -1127,13 +1111,6 @@ private void javaKVTypesNonFinalChunkWrite( startOffset = writerIndex; hasPreservedByte = true; } - - if (!keyIsNotSameType && keyWriteSerializer == null && !writeKeyClassInfo) { - keyWriteSerializer = getSerializer(key, keyClassInfoWriteCache); - } - if (!valueIsNotSameType && valueWriteSerializer == null && !writeValueClassInfo) { - valueWriteSerializer = getSerializer(value, valueClassInfoWriteCache); - } header = updateKVHeader( key, @@ -1143,55 +1120,109 @@ private void javaKVTypesNonFinalChunkWrite( header, keyIsNotSameType, valueIsNotSameType); - boolean trackingRef = fury.trackingRef(); generics.pushGenericType(keyGenericType); - if (!writeKeyClassInfo && key != null) { - writeKey( - key, - buffer, - keyWriteSerializer, - classResolver, - refResolver, - trackingRef, - keyIsNotSameType, - true); - writeKeyClassInfo = true; + if (!trackingKeyRef) { + if (key == null) { + buffer.writeByte(Fury.NULL_FLAG); + } else { + if (!keyIsNotSameType) { + ClassInfo classInfo = + classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache); + if (!writeKeyClassInfo) { + classResolver.writeClass(buffer, classInfo); + writeKeyClassInfo = true; + } + if (keyWriteSerializer == null) { + keyWriteSerializer = classInfo.getSerializer(); + } + keyWriteSerializer.write(buffer, key); + } else { + fury.writeNonRef( + buffer, key, classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache)); + } + } } else { - writeKey( - key, - buffer, - keyWriteSerializer, - classResolver, - refResolver, - trackingRef, - keyIsNotSameType, - false); + // todo 提到外面 + if (key == null) { + buffer.writeByte(Fury.NULL_FLAG); + } else { + if (!keyIsNotSameType) { + // todo key is not null, no need to write no null flag + ClassInfo classInfo = + classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache); + if (!writeKeyClassInfo) { + classResolver.writeClass(buffer, classInfo); + writeKeyClassInfo = true; + } + if (keyWriteSerializer == null) { + keyWriteSerializer = classInfo.getSerializer(); + } + writeNoNullRef(keyWriteSerializer, key, buffer, refResolver); + } else { + if (!refResolver.writeNullFlag(buffer, key)) { + fury.writeRef( + buffer, + key, + classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache)); + } + } + } } generics.popGenericType(); generics.pushGenericType(valueGenericType); - if (!writeValueClassInfo && value != null) { - writeValue( - value, - buffer, - valueWriteSerializer, - classResolver, - refResolver, - trackingRef, - valueIsNotSameType, - header, - true); - writeValueClassInfo = true; + if (!trackingValueRef) { + if (value == null) { + buffer.writeByte(Fury.NULL_FLAG); + } else { + if (!valueIsNotSameType) { + if (valueHasNull(header)) { + buffer.writeByte(Fury.NOT_NULL_VALUE_FLAG); + } + ClassInfo classInfo = + classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache); + if (!writeValueClassInfo) { + classResolver.writeClass(buffer, classInfo); + writeValueClassInfo = true; + } + if (valueWriteSerializer == null) { + valueWriteSerializer = classInfo.getSerializer(); + } + valueWriteSerializer.write(buffer, value); + } else { + fury.writeNullable( + buffer, + value, + classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache)); + } + } } else { - writeValue( - value, - buffer, - valueWriteSerializer, - classResolver, - refResolver, - trackingRef, - valueIsNotSameType, - header, - false); + if (value == null) { + buffer.writeByte(Fury.NULL_FLAG); + } else { + if (!valueIsNotSameType) { + ClassInfo classInfo = + classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache); + if (!writeValueClassInfo) { + classResolver.writeClass(buffer, classInfo); + writeValueClassInfo = true; + } + if (valueWriteSerializer == null) { + valueWriteSerializer = classInfo.getSerializer(); + } + if (!valueHasNull(header)) { + writeNoNullRef(valueWriteSerializer, value, buffer, refResolver); + } else { + fury.writeRef(buffer, value, valueWriteSerializer); + } + } else { + if (!refResolver.writeNullFlag(buffer, value)) { + fury.writeRef( + buffer, + value, + classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache)); + } + } + } } generics.popGenericType(); chunkSize++; @@ -1371,61 +1402,116 @@ protected void generalJavaChunkWrite(Fury fury, MemoryBuffer buffer, Map map) { startOffset = writerIndex; hasPreservedByte = true; } - if (!keyIsNotSameType && keyWriteSerializer == null && !writeKeyClassInfo) { - keyWriteSerializer = getSerializer(key, keyClassInfoWriteCache); - } - if (!valueIsNotSameType && valueWriteSerializer == null && !writeValueClassInfo) { - valueWriteSerializer = getSerializer(value, valueClassInfoWriteCache); - } + // if (!keyIsNotSameType && keyWriteSerializer == null && !writeKeyClassInfo) { + // keyWriteSerializer = getSerializer(key, keyClassInfoWriteCache); + // } + // if (!valueIsNotSameType && valueWriteSerializer == null && !writeValueClassInfo) { + // valueWriteSerializer = getSerializer(value, valueClassInfoWriteCache); + // } boolean trackingRef = fury.trackingRef(); header = updateKVHeader( key, trackingRef, value, trackingRef, header, keyIsNotSameType, valueIsNotSameType); - if (!writeKeyClassInfo && key != null) { - writeKey( - key, - buffer, - keyWriteSerializer, - classResolver, - refResolver, - trackingRef, - keyIsNotSameType, - true); - writeKeyClassInfo = true; + if (!trackingRef) { + if (key == null) { + buffer.writeByte(Fury.NULL_FLAG); + } else { + if (!keyIsNotSameType) { + ClassInfo classInfo = + classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache); + if (!writeKeyClassInfo) { + classResolver.writeClass(buffer, classInfo); + writeKeyClassInfo = true; + } + if (keyWriteSerializer == null) { + keyWriteSerializer = classInfo.getSerializer(); + } + keyWriteSerializer.write(buffer, key); + } else { + fury.writeNonRef( + buffer, key, classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache)); + } + } } else { - writeKey( - key, - buffer, - keyWriteSerializer, - classResolver, - refResolver, - trackingRef, - keyIsNotSameType, - false); + // todo 提到外面 + if (key == null) { + buffer.writeByte(Fury.NULL_FLAG); + } else { + if (!keyIsNotSameType) { + // todo key is not null, no need to write no null flag + ClassInfo classInfo = + classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache); + if (!writeKeyClassInfo) { + classResolver.writeClass(buffer, classInfo); + writeKeyClassInfo = true; + } + if (keyWriteSerializer == null) { + keyWriteSerializer = classInfo.getSerializer(); + } + writeNoNullRef(keyWriteSerializer, key, buffer, refResolver); + } else { + if (!refResolver.writeNullFlag(buffer, key)) { + fury.writeRef( + buffer, + key, + classResolver.getClassInfo(key.getClass(), keyClassInfoWriteCache)); + } + } + } } - if (!writeValueClassInfo && value != null) { - writeValue( - value, - buffer, - valueWriteSerializer, - classResolver, - refResolver, - trackingRef, - valueIsNotSameType, - header, - true); - writeValueClassInfo = true; + if (!trackingRef) { + if (value == null) { + buffer.writeByte(Fury.NULL_FLAG); + } else { + if (!valueIsNotSameType) { + if (valueHasNull(header)) { + buffer.writeByte(Fury.NOT_NULL_VALUE_FLAG); + } + ClassInfo classInfo = + classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache); + if (!writeValueClassInfo) { + classResolver.writeClass(buffer, classInfo); + writeValueClassInfo = true; + } + if (valueWriteSerializer == null) { + valueWriteSerializer = classInfo.getSerializer(); + } + valueWriteSerializer.write(buffer, value); + } else { + fury.writeNullable( + buffer, + value, + classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache)); + } + } } else { - writeValue( - value, - buffer, - valueWriteSerializer, - classResolver, - refResolver, - trackingRef, - valueIsNotSameType, - header, - false); + if (value == null) { + buffer.writeByte(Fury.NULL_FLAG); + } else { + if (!valueIsNotSameType) { + ClassInfo classInfo = + classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache); + if (!writeValueClassInfo) { + classResolver.writeClass(buffer, classInfo); + writeValueClassInfo = true; + } + if (valueWriteSerializer == null) { + valueWriteSerializer = classInfo.getSerializer(); + } + if (!valueHasNull(header)) { + writeNoNullRef(valueWriteSerializer, value, buffer, refResolver); + } else { + fury.writeRef(buffer, value, valueWriteSerializer); + } + } else { + if (!refResolver.writeNullFlag(buffer, value)) { + fury.writeRef( + buffer, + value, + classResolver.getClassInfo(value.getClass(), valueClassInfoWriteCache)); + } + } + } } chunkSize++; } else {