fix meta string utf-8 xlang decoding

apache · Apr 16, 2024 · 0688b56 · 0688b56
1 parent 662818a
commit 0688b56
Show file tree

Hide file tree

Showing 4 changed files with 10 additions and 5 deletions.
diff --git a/go/fury/type.go b/go/fury/type.go
@@ -557,7 +557,7 @@ func (r *typeResolver) writeMetaString(buffer *ByteBuffer, str string) error {
 		if _, err := h.Write([]byte(str)); err != nil {
 			return err
 		}
-		hash := int64(h.Sum64())
+		hash := int64(h.Sum64() & 0xffffffffffffff00)
 		buffer.WriteInt64(hash)
 		if len(str) > MaxInt16 {
 			return fmt.Errorf("too long string: %s", str)

diff --git a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassInfo.java b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassInfo.java
@@ -20,6 +20,7 @@
 package org.apache.fury.resolver;
 
 import org.apache.fury.config.Language;
+import org.apache.fury.meta.MetaString.Encoding;
 import org.apache.fury.meta.MetaStringEncoder;
 import org.apache.fury.serializer.Serializer;
 import org.apache.fury.util.Preconditions;
@@ -76,7 +77,7 @@ public class ClassInfo {
     if (cls != null && classResolver.getFury().getLanguage() != Language.JAVA) {
       this.fullClassNameBytes =
           metaStringResolver.getOrCreateMetaStringBytes(
-              new MetaStringEncoder('.', '_').encode(cls.getName()));
+              new MetaStringEncoder('.', '_').encode(cls.getName(), Encoding.UTF_8));
     } else {
       this.fullClassNameBytes = null;
     }
@@ -98,7 +99,7 @@ public class ClassInfo {
     if (tag != null) {
       this.typeTagBytes =
           metaStringResolver.getOrCreateMetaStringBytes(
-              new MetaStringEncoder('.', '_').encode(tag));
+              new MetaStringEncoder('.', '_').encode(tag, Encoding.UTF_8));
     } else {
       this.typeTagBytes = null;
     }

diff --git a/python/pyfury/_fury.py b/python/pyfury/_fury.py
@@ -100,7 +100,9 @@ class MetaStringBytes:
     def __init__(self, data, hashcode=None):
         self.data = data
         self.length = len(data)
-        self.hashcode = hashcode or mmh3.hash_buffer(data, 47)[0]
+        if hashcode is None:
+            hashcode = mmh3.hash_buffer(data, 47)[0] & 0xffffffffffffff00
+        self.hashcode = hashcode
         self.dynamic_write_string_id = DEFAULT_DYNAMIC_WRITE_STRING_ID
 
     def __eq__(self, other):

diff --git a/python/pyfury/_serialization.pyx b/python/pyfury/_serialization.pyx
@@ -759,7 +759,9 @@ cdef class MetaStringBytes:
     def __init__(self, data, hashcode=None):
         self.data = data
         self.length = len(data)
-        self.hashcode = hashcode or mmh3.hash_buffer(data, 47)[0]
+        if hashcode is None:
+            hashcode = mmh3.hash_buffer(data, 47)[0] & 0xffffffffffffff00
+        self.hashcode = hashcode
         self.dynamic_write_string_id = DEFAULT_DYNAMIC_WRITE_STRING_ID
 
     def __eq__(self, other):