[CI] Enable Fury CI

.github/workflows/ci.yml

@@ -26,8 +26,14 @@ jobs:
         with:
           java-version: ${{ matrix.java-version }}
           distribution: 'temurin'
+      - name: Install bazel
+        run: ./ci/run_ci.sh install_bazel
+      - name: Install python
+        run: ./ci/run_ci.sh install_python
+      - name: Install pyfury
+        run: ./ci/run_ci.sh install_pyfury
       - name: Run CI with Maven
-        run: ./ci/run_ci.sh java${{ matrix.java-version }}
+        run: PATH="$HOME/miniconda/bin:$PATH" ./ci/run_ci.sh java${{ matrix.java-version }}
   javascript:
     name: Javascript CI
     runs-on: ubuntu-latest

ci/run_ci.sh

@@ -6,6 +6,25 @@ ROOT="$(git rev-parse --show-toplevel)"
 echo "Root path: $ROOT, home path: $HOME"
 cd "$ROOT"
 
+install_python() {
+  wget -q https://repo.anaconda.com/miniconda/Miniconda3-py38_23.5.2-0-Linux-x86_64.sh -O Miniconda3.sh
+  bash Miniconda3.sh -b -p $HOME/miniconda && rm -f miniconda.*
+  which python
+  echo "Python version $(python -V)"
+}
+
+install_pyfury() {
+  export PATH="$HOME/miniconda/bin:$PATH"
+  echo "Python version $(python -V)"
+  pip install pyarrow==4.0.0 Cython wheel numpy pytest
+  pushd "$ROOT/python"
+  pip list
+  export PATH=~/bin:$PATH
+  echo "Install pyfury"
+  pip install -v -e .
+  popd
+}
+
 install_bazel() {
   if command -v java >/dev/null; then
     echo "existing bazel location $(which bazel)"
@@ -107,12 +126,9 @@ case $1 in
       echo "Executing fury c++ tests succeeds"
     ;;
     python)
-      pip install pyarrow==4.0.0 Cython wheel numpy pytest
+      install_pyfury
+      pip install pandas
       cd "$ROOT/python"
-      pip list
-      export PATH=~/bin:$PATH
-      echo "Install pyfury"
-      pip install -v -e .
       echo "Executing fury python tests"
       pytest -v -s --durations=60 pyfury/tests
       testcode=$?

java/pom.xml

@@ -18,6 +18,36 @@
   <artifactId>fury-parent</artifactId>
   <packaging>pom</packaging>
   <version>0.1.0-SNAPSHOT</version>
+  <name>Fury Project Parent POM</name>
+  <description>A blazing fast multi-language serialization framework powered by jit and zero-copy.</description>
+  <url>https://github.com/alipay/fury</url>
+  <licenses>
+    <license>
+      <name>The Apache License, Version 2.0</name>
+      <url>https://www.apache.org/licenses/LICENSE-2.0.txt</url>
+    </license>
+  </licenses>
+  <scm>
+    <url>https://github.com/alipay/fury</url>
+    <connection>[email protected]:alipay/fury.git</connection>
+    <developerConnection>scm:git:ssh://github.com:alipay/fury.git</developerConnection>
+  </scm>
+  <developers>
+    <developer>
+      <organizationUrl>https://furyio.org</organizationUrl>
+    </developer>
+  </developers>
+  <distributionManagement>
+    <snapshotRepository>
+      <id>ossrh</id>
+      <url>https://s01.oss.sonatype.org/content/repositories/snapshots</url>
+    </snapshotRepository>
+    <repository>
+      <id>ossrh</id>
+      <url>https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/</url>
+    </repository>
+  </distributionManagement>
+
   <modules>
     <module>fury-format</module>
     <module>fury-core</module>

python/pyfury/_fury.py

@@ -126,7 +126,7 @@ def __repr__(self):
 
 class ClassResolver:
     __slots__ = (
-        "fury_",
+        "fury",
         "_type_id_to_class",
         "_type_id_to_serializer",
         "_type_id_and_cls_to_serializer",
@@ -151,8 +151,8 @@ class ClassResolver:
     _type_id_and_cls_to_serializer: Dict[Tuple[int, type], Serializer]
     _classes_info: Dict[type, "ClassInfo"]
 
-    def __init__(self, fury_):
-        self.fury_ = fury_
+    def __init__(self, fury):
+        self.fury = fury
         self._type_id_to_class = dict()
         self._type_id_to_serializer = dict()
         self._type_id_and_cls_to_serializer = dict()
@@ -249,18 +249,16 @@ def register_class_tag(self, cls: type, type_tag: str = None):
         serialization."""
         from pyfury._struct import ComplexObjectSerializer
 
-        self.register_serializer(
-            cls, ComplexObjectSerializer(self.fury_, cls, type_tag)
-        )
+        self.register_serializer(cls, ComplexObjectSerializer(self.fury, cls, type_tag))
 
     def _add_serializer(self, cls: type, serializer=None, serializer_cls=None):
         if serializer_cls:
-            serializer = serializer_cls(self.fury_, cls)
+            serializer = serializer_cls(self.fury, cls)
         self.register_serializer(cls, serializer)
 
     def _add_x_lang_serializer(self, cls: type, serializer=None, serializer_cls=None):
         if serializer_cls:
-            serializer = serializer_cls(self.fury_, cls)
+            serializer = serializer_cls(self.fury, cls)
         type_id = serializer.get_xtype_id()
         from pyfury._serializer import NOT_SUPPORT_CROSS_LANGUAGE
 
@@ -313,10 +311,10 @@ def _add_default_serializers(self):
         from pyfury import PickleCacheSerializer, PickleStrongCacheSerializer
 
         self._add_serializer(
-            PickleStrongCacheStub, serializer=PickleStrongCacheSerializer(self.fury_)
+            PickleStrongCacheStub, serializer=PickleStrongCacheSerializer(self.fury)
         )
         self._add_serializer(
-            PickleCacheStub, serializer=PickleCacheSerializer(self.fury_)
+            PickleCacheStub, serializer=PickleCacheSerializer(self.fury)
         )
         try:
             import pyarrow as pa
@@ -334,17 +332,17 @@ def _add_default_serializers(self):
         for typecode in PyArraySerializer.typecode_dict.keys():
             self._add_serializer(
                 array.array,
-                serializer=PyArraySerializer(self.fury_, array.array, typecode),
+                serializer=PyArraySerializer(self.fury, array.array, typecode),
             )
             self._add_serializer(
                 PyArraySerializer.typecodearray_type[typecode],
-                serializer=PyArraySerializer(self.fury_, array.array, typecode),
+                serializer=PyArraySerializer(self.fury, array.array, typecode),
             )
         if np:
             for dtype in Numpy1DArraySerializer.dtypes_dict.keys():
                 self._add_serializer(
                     np.ndarray,
-                    serializer=Numpy1DArraySerializer(self.fury_, array.array, dtype),
+                    serializer=Numpy1DArraySerializer(self.fury, array.array, dtype),
                 )
 
     def get_serializer(self, cls: type = None, type_id: int = None, obj=None):
@@ -423,7 +421,7 @@ def _create_serializer(self, cls):
             ):
                 if classinfo_ is None or classinfo_.class_id == NO_CLASS_ID:
                     logger.info("Class %s not registered", cls)
-                serializer = type(class_info.serializer)(self.fury_, cls)
+                serializer = type(class_info.serializer)(self.fury, cls)
                 break
         else:
             if dataclasses.is_dataclass(cls):
@@ -432,9 +430,9 @@ def _create_serializer(self, cls):
                 logger.info("Class %s not registered", cls)
                 from pyfury import DataClassSerializer
 
-                serializer = DataClassSerializer(self.fury_, cls)
+                serializer = DataClassSerializer(self.fury, cls)
             else:
-                serializer = PickleSerializer(self.fury_, cls)
+                serializer = PickleSerializer(self.fury, cls)
         return serializer
 
     def write_classinfo(self, buffer: Buffer, classinfo: ClassInfo):
@@ -1015,6 +1013,6 @@ def __init__(self, buf):
 
     def load(self):
         raise ValueError(
-            f"pickle is not allowed when secure mode enabled, Please register"
-            f"the class or pass unsupported_callback"
+            "pickle is not allowed when secure mode enabled, Please register"
+            "the class or pass unsupported_callback"
         )

python/pyfury/_serialization.pyx

@@ -12,7 +12,8 @@ import sys
 from typing import TypeVar, Union, Iterable, get_type_hints
 
 from pyfury._util import get_bit, set_bit, clear_bit
-from pyfury._fury import Language, OpaqueObject, _PicklerStub, _UnpicklerStub, _ENABLE_SECURITY_MODE_FORCIBLY
+from pyfury._fury import Language, OpaqueObject
+from pyfury._fury import _PicklerStub, _UnpicklerStub, _ENABLE_SECURITY_MODE_FORCIBLY
 from pyfury.error import ClassNotCompatibleError
 from pyfury.lib import mmh3
 from pyfury.type import is_primitive_type, FuryType, Int8Type, Int16Type, Int32Type, \
@@ -269,7 +270,7 @@ cdef class PickleCacheStub:
 @cython.final
 cdef class ClassResolver:
     cdef:
-        readonly Fury fury_
+        readonly Fury fury
         dict _type_id_to_class  # Dict[int, type]
         dict _type_id_to_serializer  # Dict[int, Serializer]
         dict _type_id_and_cls_to_serializer  # Dict[Tuple[int, type], Serializer]
@@ -304,8 +305,8 @@ cdef class ClassResolver:
         set _classname_set
         set _enum_str_set
 
-    def __init__(self, fury_):
-        self.fury_ = fury_
+    def __init__(self, fury):
+        self.fury = fury
         self._type_id_to_class = dict()
         self._type_id_to_serializer = dict()
         self._type_id_and_cls_to_serializer = dict()
@@ -398,7 +399,7 @@ cdef class ClassResolver:
         """Register class with given type tag which will be used for cross-language
         serialization."""
         self.register_serializer(
-            cls, ComplexObjectSerializer(self.fury_, cls, type_tag)
+            cls, ComplexObjectSerializer(self.fury, cls, type_tag)
         )
 
     def _add_serializer(
@@ -407,15 +408,15 @@ cdef class ClassResolver:
             serializer=None,
             serializer_cls=None):
         if serializer_cls:
-            serializer = serializer_cls(self.fury_, cls)
+            serializer = serializer_cls(self.fury, cls)
         self.register_serializer(cls, serializer)
 
     def _add_x_lang_serializer(self,
                                cls: Union[type, TypeVar],
                                serializer=None,
                                serializer_cls=None):
         if serializer_cls:
-            serializer = serializer_cls(self.fury_, cls)
+            serializer = serializer_cls(self.fury, cls)
         type_id = serializer.get_xtype_id()
 
         assert type_id != NOT_SUPPORT_CROSS_LANGUAGE
@@ -464,9 +465,9 @@ cdef class ClassResolver:
         from pyfury import PickleCacheSerializer, PickleStrongCacheSerializer
 
         self._add_serializer(PickleStrongCacheStub,
-                             serializer=PickleStrongCacheSerializer(self.fury_))
+                             serializer=PickleStrongCacheSerializer(self.fury))
         self._add_serializer(PickleCacheStub,
-                             serializer=PickleCacheSerializer(self.fury_))
+                             serializer=PickleCacheSerializer(self.fury))
 
         try:
             import pyarrow as pa
@@ -484,17 +485,17 @@ cdef class ClassResolver:
         for typecode in PyArraySerializer.typecode_dict.keys():
             self._add_serializer(
                 array.array,
-                serializer=PyArraySerializer(self.fury_, array.array, typecode),
+                serializer=PyArraySerializer(self.fury, array.array, typecode),
             )
             self._add_serializer(
                 PyArraySerializer.typecodearray_type[typecode],
-                serializer=PyArraySerializer(self.fury_, array.array, typecode),
+                serializer=PyArraySerializer(self.fury, array.array, typecode),
             )
         if np:
             for dtype in Numpy1DArraySerializer.dtypes_dict.keys():
                 self._add_serializer(
                     np.ndarray,
-                    serializer=Numpy1DArraySerializer(self.fury_, array.array, dtype),
+                    serializer=Numpy1DArraySerializer(self.fury, array.array, dtype),
                 )
 
     cpdef inline Serializer get_serializer(self, cls=None, type_id=None, obj=None):
@@ -575,17 +576,17 @@ cdef class ClassResolver:
             ):
                 if classinfo_ is None or classinfo_.class_id == NO_CLASS_ID:
                     logger.info("Class %s not registered", cls)
-                serializer = type(class_info.serializer)(self.fury_, cls)
+                serializer = type(class_info.serializer)(self.fury, cls)
                 break
         else:
             if dataclasses.is_dataclass(cls):
                 if classinfo_ is None or classinfo_.class_id == NO_CLASS_ID:
                     logger.info("Class %s not registered", cls)
                 from pyfury import DataClassSerializer
 
-                serializer = DataClassSerializer(self.fury_, cls)
+                serializer = DataClassSerializer(self.fury, cls)
             else:
-                serializer = PickleSerializer(self.fury_, cls)
+                serializer = PickleSerializer(self.fury, cls)
         return serializer
 
     cpdef inline write_classinfo(self, Buffer buffer, ClassInfo classinfo):
@@ -1307,12 +1308,12 @@ cdef class SerializationContext:
 
 
 cdef class Serializer:
-    cdef readonly Fury fury_
+    cdef readonly Fury fury
     cdef readonly object type_
     cdef public c_bool need_to_write_ref
 
-    def __init__(self, fury_, type_: Union[type, TypeVar]):
-        self.fury = fury_
+    def __init__(self, fury, type_: Union[type, TypeVar]):
+        self.fury = fury
         self.type_ = type_
         self.need_to_write_ref = not is_primitive_type(type_)
 
@@ -1362,8 +1363,8 @@ cdef class CrossLanguageCompatibleSerializer(Serializer):
     cpdef read(self, Buffer buffer):
         raise NotImplementedError
 
-    def __init__(self, fury_, type_):
-        super().__init__(fury_, type_)
+    def __init__(self, fury, type_):
+        super().__init__(fury, type_)
 
     cpdef xwrite(self, Buffer buffer, value):
         self.write(buffer, value)
@@ -1965,8 +1966,8 @@ cdef class PyArraySerializer(CrossLanguageCompatibleSerializer):
     cdef int8_t itemsize
     cdef int16_t type_id
 
-    def __init__(self, fury_, type_, str typecode):
-        super().__init__(fury_, type_)
+    def __init__(self, fury, type_, str typecode):
+        super().__init__(fury, type_)
         self.typecode = typecode
         self.itemsize, self.type_id = PyArraySerializer.typecode_dict[self.typecode]
 
@@ -2025,8 +2026,8 @@ cdef class Numpy1DArraySerializer(CrossLanguageCompatibleSerializer):
     cdef int8_t itemsize
     cdef int16_t type_id
 
-    def __init__(self, fury_, type_, dtype):
-        super().__init__(fury_, type_)
+    def __init__(self, fury, type_, dtype):
+        super().__init__(fury, type_)
         self.dtype = dtype
         self.itemsize, self.typecode, self.type_id = _np_dtypes_dict[self.dtype]
 
@@ -2059,10 +2060,10 @@ cdef class Numpy1DArraySerializer(CrossLanguageCompatibleSerializer):
         return self.fury.handle_unsupported_read(buffer)
 
 
-cdef _get_hash(Fury fury_, list field_names, dict type_hints):
+cdef _get_hash(Fury fury, list field_names, dict type_hints):
     from pyfury._struct import StructHashVisitor
 
-    visitor = StructHashVisitor(fury_)
+    visitor = StructHashVisitor(fury)
     for index, key in enumerate(field_names):
         infer_field(key, type_hints[key], visitor, types_path=[])
     hash_ = visitor.get_hash()
@@ -2077,15 +2078,15 @@ cdef class ComplexObjectSerializer(Serializer):
     cdef list _serializers
     cdef int32_t _hash
 
-    def __init__(self, fury_, clz: type, type_tag: str):
-        super().__init__(fury_, clz)
+    def __init__(self, fury, clz: type, type_tag: str):
+        super().__init__(fury, clz)
         self._type_tag = type_tag
         self._type_hints = get_type_hints(clz)
         self._field_names = sorted(self._type_hints.keys())
         self._serializers = [None] * len(self._field_names)
         from pyfury._struct import ComplexTypeVisitor
 
-        visitor = ComplexTypeVisitor(fury_)
+        visitor = ComplexTypeVisitor(fury)
         for index, key in enumerate(self._field_names):
             serializer = infer_field(key, self._type_hints[key], visitor, types_path=[])
             self._serializers[index] = serializer
@@ -2148,7 +2149,6 @@ cdef class ComplexObjectSerializer(Serializer):
         return obj
 
 
-
 @cython.final
 cdef class EnumSerializer(Serializer):
     @classmethod