Repository URL to install this package:
|
Version:
2.7.1 ▾
|
from collections.abc import Iterable
from typing import Callable, Generic, TypeVar
T = TypeVar("T")
_ENCODING_VERSION: int = 1
__all__ = ["AppendingByteSerializer"]
#######################################
# Helper classes
#######################################
class BytesWriter:
def __init__(self, preallocate_size: int) -> None:
self._data = bytearray(preallocate_size)
def write_uint64(self, i: int) -> None:
self._data.extend(i.to_bytes(8, byteorder="big", signed=False))
def write_str(self, s: str) -> None:
payload = s.encode("utf-8")
self.write_bytes(payload)
def write_bytes(self, b: bytes) -> None:
self.write_uint64(len(b))
self._data.extend(b)
def to_bytes(self) -> bytes:
return bytes(self._data)
class BytesReader:
def __init__(self, data: bytes) -> None:
self._data = data
self._i = 0
def is_finished(self) -> bool:
return len(self._data) == self._i
def read_uint64(self) -> int:
result = int.from_bytes(
self._data[self._i : self._i + 8], byteorder="big", signed=False
)
self._i += 8
return result
def read_str(self) -> str:
return self.read_bytes().decode("utf-8")
def read_bytes(self) -> bytes:
size = self.read_uint64()
result = self._data[self._i : self._i + size]
self._i += size
return result
#######################################
# AppendingByteSerializer
#######################################
class AppendingByteSerializer(Generic[T]):
"""
Provides efficient serialization and deserialization of list of bytes
Note that this does not provide any guarantees around byte order
"""
_serialize_fn: Callable[[BytesWriter, T], None]
_writer: BytesWriter
_preallocate_size: int
def __init__(
self,
*,
serialize_fn: Callable[[BytesWriter, T], None],
preallocate_size: int = 0,
) -> None:
self._serialize_fn = serialize_fn
self._preallocate_size = preallocate_size
self.clear()
def clear(self) -> None:
self._writer = BytesWriter(preallocate_size=self._preallocate_size)
# First 8-bytes are for version
self._writer.write_uint64(_ENCODING_VERSION)
def append(self, data: T) -> None:
self._serialize_fn(self._writer, data)
def extend(self, elems: Iterable[T]) -> None:
for elem in elems:
self.append(elem)
def to_bytes(self) -> bytes:
return self._writer.to_bytes()
@staticmethod
def to_list(data: bytes, *, deserialize_fn: Callable[[BytesReader], T]) -> list[T]:
reader = BytesReader(data)
assert reader.read_uint64() == _ENCODING_VERSION
result: list[T] = []
while not reader.is_finished():
result.append(deserialize_fn(reader))
return result