lmcache.experimental.storage_backend.naive_serde package#

Submodules#

lmcache.experimental.storage_backend.naive_serde.cachegen_basics module#

class lmcache.experimental.storage_backend.naive_serde.cachegen_basics.CacheGenConfig(nlayers: int, kspecs: List[lmcache.storage_backend.serde.cachegen_basics.QuantizationSpec], vspecs: List[lmcache.storage_backend.serde.cachegen_basics.QuantizationSpec])[source]#

static from_model_name(model_name: str) → CacheGenConfig[source]#

kspecs: List[QuantizationSpec]#

nlayers: int#

vspecs: List[QuantizationSpec]#

lmcache.experimental.storage_backend.naive_serde.cachegen_decoder module#

class lmcache.experimental.storage_backend.naive_serde.cachegen_decoder.CacheGenDeserializer(config: LMCacheEngineConfig, metadata: LMCacheEngineMetadata, memory_allocator: MemoryAllocatorInterface)[source]#

Bases: Deserializer

deserialize(buffer_memory_obj: BytesBufferMemoryObj) → MemoryObj | None[source]#

Deserialize/decompress the memory object.

Input:: memory_obj: the memory object to be deserialized/decompressed.

Returns:: the deserialized/decompressed memory object. None: if the memory allocation fails.
Return type:: MemoryObj

get_output_buffer(nlayers: int, nchannels: int, ntokens: int)[source]#

make_key_bins(config: CacheGenConfig) → Tensor[source]#

make_value_bins(config: CacheGenConfig) → Tensor[source]#

lmcache.experimental.storage_backend.naive_serde.cachegen_encoder module#

class lmcache.experimental.storage_backend.naive_serde.cachegen_encoder.CacheGenSerializer(config: LMCacheEngineConfig, metadata: LMCacheEngineMetadata, memory_allocator: MemoryAllocatorInterface)[source]#

Bases: Serializer

make_key_bins(config: CacheGenConfig) → Tensor[source]#

make_value_bins(config: CacheGenConfig) → Tensor[source]#

serialize(memory_obj: MemoryObj) → BytesBufferMemoryObj[source]#

Serialize a KV_BLOB MemoryObj to CACHEGEN_BINARY MemoryObj.

Input:: memory_obj: the memory object to be serialized.

Returns:: the serialized binary memory object.
Return type:: MemoryObj

lmcache.experimental.storage_backend.naive_serde.kivi_serde module#

class lmcache.experimental.storage_backend.naive_serde.kivi_serde.KIVIDeserializer(memory_allocator: MemoryAllocatorInterface)[source]#

Bases: Deserializer

deserialize(memory_obj: MemoryObj) → MemoryObj[source]#

Deserialize/decompress the memory object.

Input:: memory_obj: the memory object to be deserialized/decompressed.

Returns:: the deserialized/decompressed memory object. None: if the memory allocation fails.
Return type:: MemoryObj

class lmcache.experimental.storage_backend.naive_serde.kivi_serde.KIVISerializer(memory_allocator: MemoryAllocatorInterface)[source]#

Bases: Serializer

serialize(memory_obj: MemoryObj) → MemoryObj[source]#

Serialize/compress the memory object.

Input:: memory_obj: the memory object to be serialized/compressed.

Returns:: the serialized/compressed memory object.
Return type:: MemoryObj

lmcache.experimental.storage_backend.naive_serde.naive_serde module#

class lmcache.experimental.storage_backend.naive_serde.naive_serde.NaiveDeserializer[source]#

Bases: Deserializer

deserialize(memory_obj: MemoryObj) → MemoryObj[source]#

Deserialize/decompress the memory object.

Input:: memory_obj: the memory object to be deserialized/decompressed.

Returns:: the deserialized/decompressed memory object. None: if the memory allocation fails.
Return type:: MemoryObj

class lmcache.experimental.storage_backend.naive_serde.naive_serde.NaiveSerializer(memory_allocator)[source]#

Bases: Serializer

serialize(memory_obj: MemoryObj) → MemoryObj[source]#

Serialize/compress the memory object.

Input:: memory_obj: the memory object to be serialized/compressed.

Returns:: the serialized/compressed memory object.
Return type:: MemoryObj

lmcache.experimental.storage_backend.naive_serde.serde module#

class lmcache.experimental.storage_backend.naive_serde.serde.Deserializer[source]#

abstract deserialize(memory_obj: MemoryObj) → MemoryObj | None[source]#

Deserialize/decompress the memory object.

Input:: memory_obj: the memory object to be deserialized/decompressed.

Returns:: the deserialized/decompressed memory object. None: if the memory allocation fails.
Return type:: MemoryObj

class lmcache.experimental.storage_backend.naive_serde.serde.Serializer[source]#

abstract serialize(memory_obj: MemoryObj) → MemoryObj[source]#

Serialize/compress the memory object.

Input:: memory_obj: the memory object to be serialized/compressed.

Returns:: the serialized/compressed memory object.
Return type:: MemoryObj

Module contents#

lmcache.experimental.storage_backend.naive_serde.CreateSerde(serde_type: str, memory_allocator: MemoryAllocatorInterface, metadata: LMCacheEngineMetadata, config: LMCacheEngineConfig) → Tuple[Serializer, Deserializer][source]#

class lmcache.experimental.storage_backend.naive_serde.Deserializer[source]#

abstract deserialize(memory_obj: MemoryObj) → MemoryObj | None[source]#

Deserialize/decompress the memory object.

Input:: memory_obj: the memory object to be deserialized/decompressed.

Returns:: the deserialized/decompressed memory object. None: if the memory allocation fails.
Return type:: MemoryObj

class lmcache.experimental.storage_backend.naive_serde.KIVIDeserializer(memory_allocator: MemoryAllocatorInterface)[source]#

Bases: Deserializer

deserialize(memory_obj: MemoryObj) → MemoryObj[source]#

Deserialize/decompress the memory object.

Input:: memory_obj: the memory object to be deserialized/decompressed.

Returns:: the deserialized/decompressed memory object. None: if the memory allocation fails.
Return type:: MemoryObj

class lmcache.experimental.storage_backend.naive_serde.KIVISerializer(memory_allocator: MemoryAllocatorInterface)[source]#

Bases: Serializer

serialize(memory_obj: MemoryObj) → MemoryObj[source]#

Serialize/compress the memory object.

Input:: memory_obj: the memory object to be serialized/compressed.

Returns:: the serialized/compressed memory object.
Return type:: MemoryObj

class lmcache.experimental.storage_backend.naive_serde.Serializer[source]#

abstract serialize(memory_obj: MemoryObj) → MemoryObj[source]#

Serialize/compress the memory object.

Input:: memory_obj: the memory object to be serialized/compressed.

Returns:: the serialized/compressed memory object.
Return type:: MemoryObj