lmcache.storage_backend.evictor package#

Submodules#

lmcache.storage_backend.evictor.base_evictor module#

class lmcache.storage_backend.evictor.base_evictor.BaseEvictor[source]#

Interface for cache evictor

get_size(kv_obj: Tensor | bytes) → float[source]#

Get the size of the kv cache

Input:: kv_obj: kv cache

Returns:: the size of the cache (in GB)

abstract update_on_get(key: CacheEngineKey | str, cache_dict: OrderedDict) → None[source]#

Update cache_dict when a cache is used is used

Input:: key: a CacheEngineKey cache_dict: a dict consists of current cache

abstract update_on_put(cache_dict: OrderedDict, kv_obj: Tensor | bytes) → Tuple[List[CacheEngineKey | str], PutStatus][source]#

Evict cache when a new cache comes and the storage is full

Input:: cache_dict: a dict consists of current cache kv_obj: the new kv cache to be injected

Returns:: return a key to be evicted

class lmcache.storage_backend.evictor.base_evictor.DummyEvictor[source]#

Bases: BaseEvictor

update_on_get(key: CacheEngineKey | str, cache_dict: OrderedDict) → None[source]#

Update cache_dict when a cache is used is used

Input:: key: a CacheEngineKey cache_dict: a dict consists of current cache

update_on_put(cache_dict: OrderedDict, kv_obj: Tensor | bytes)[source]#

Evict cache when a new cache comes and the storage is full

Input:: cache_dict: a dict consists of current cache kv_obj: the new kv cache to be injected

Returns:: return a key to be evicted

class lmcache.storage_backend.evictor.base_evictor.PutStatus(value, names=<not given>, *values, module=None, qualname=None, type=None, start=1, boundary=None)[source]#

Bases: Enum

ILLEGAL = 2#

LEGAL = 1#

lmcache.storage_backend.evictor.lru_evictor module#

class lmcache.storage_backend.evictor.lru_evictor.LRUEvictor(max_cache_size: float = 10.0)[source]#

Bases: BaseEvictor

LRU cache evictor

update_on_get(key: CacheEngineKey | str, cache_dict: OrderedDict) → None[source]#

Evict cache when a new cache comes and the storage is full

Input:: key: a CacheEngineKey or a str cache_dict: a dict consists of current cache

update_on_put(cache_dict: OrderedDict, kv_obj: Tensor | bytes)[source]#

Evict cache when a new cache comes and the storage is full

Input:: cache_dict: a dict consists of current cache kv_obj: the new kv cache to be injected

Returns:: a list of keys to be evicted
Return type:: evict_keys

Module contents#

class lmcache.storage_backend.evictor.DummyEvictor[source]#

Bases: BaseEvictor

update_on_get(key: CacheEngineKey | str, cache_dict: OrderedDict) → None[source]#

Update cache_dict when a cache is used is used

Input:: key: a CacheEngineKey cache_dict: a dict consists of current cache

update_on_put(cache_dict: OrderedDict, kv_obj: Tensor | bytes)[source]#

Evict cache when a new cache comes and the storage is full

Input:: cache_dict: a dict consists of current cache kv_obj: the new kv cache to be injected

Returns:: return a key to be evicted

class lmcache.storage_backend.evictor.LRUEvictor(max_cache_size: float = 10.0)[source]#

Bases: BaseEvictor

LRU cache evictor

update_on_get(key: CacheEngineKey | str, cache_dict: OrderedDict) → None[source]#

Evict cache when a new cache comes and the storage is full

Input:: key: a CacheEngineKey or a str cache_dict: a dict consists of current cache

update_on_put(cache_dict: OrderedDict, kv_obj: Tensor | bytes)[source]#

Evict cache when a new cache comes and the storage is full

Input:: cache_dict: a dict consists of current cache kv_obj: the new kv cache to be injected

Returns:: a list of keys to be evicted
Return type:: evict_keys