lmcache.storage_backend.evictor package#

Submodules#

lmcache.storage_backend.evictor.base_evictor module#

class lmcache.storage_backend.evictor.base_evictor.BaseEvictor[source]#

Interface for cache evictor

get_size(kv_obj: Tensor | bytes | KVObj) → int[source]#

Get the size of the kv cache

Input:

kv_obj: kv cache

Returns:

the size of the cache (in bytes)

abstract update_on_get(key: CacheEngineKey | str, cache_dict: OrderedDict) → None[source]#

Update cache_dict when a cache is used is used

Input:

key: a CacheEngineKey cache_dict: a dict consists of current cache

abstract update_on_put(cache_dict: OrderedDict, cache_size: int) → Tuple[List[CacheEngineKey | str], PutStatus][source]#

Evict cache when a new cache comes and the storage is full

Input:

cache_dict: a dict consists of current cache kv_obj: the new kv cache to be injected

Returns:

return a key to be evicted

class lmcache.storage_backend.evictor.base_evictor.DummyEvictor[source]#

Bases: BaseEvictor

update_on_get(key: CacheEngineKey | str, cache_dict: OrderedDict) → None[source]#

Update cache_dict when a cache is used is used

Input:

key: a CacheEngineKey cache_dict: a dict consists of current cache

update_on_put(cache_dict: OrderedDict, cache_size: int)[source]#

Evict cache when a new cache comes and the storage is full

Input:

cache_dict: a dict consists of current cache kv_obj: the new kv cache to be injected

Returns:

return a key to be evicted

class lmcache.storage_backend.evictor.base_evictor.PutStatus(value, names=<not given>, *values, module=None, qualname=None, type=None, start=1, boundary=None)[source]#

Bases: Enum

ILLEGAL = 2#
LEGAL = 1#

lmcache.storage_backend.evictor.lru_evictor module#

class lmcache.storage_backend.evictor.lru_evictor.LRUEvictor(max_cache_size: float = 10.0)[source]#

Bases: BaseEvictor

LRU cache evictor

update_on_get(key: CacheEngineKey | str, cache_dict: OrderedDict) → None[source]#

Update cache recency when a cache is used

Input:

key: a CacheEngineKey or a str cache_dict: a dict consists of current cache

update_on_put(cache_dict: OrderedDict, cache_size: int)[source]#

Evict cache when a new cache comes and the storage is full

Input:

cache_dict: a dict consists of current cache kv_obj: the new kv cache to be injected

Returns:

a list of keys to be evicted

Return type:

evict_keys

Module contents#

class lmcache.storage_backend.evictor.DummyEvictor[source]#

Bases: BaseEvictor

update_on_get(key: CacheEngineKey | str, cache_dict: OrderedDict) → None[source]#

Update cache_dict when a cache is used is used

Input:

key: a CacheEngineKey cache_dict: a dict consists of current cache

update_on_put(cache_dict: OrderedDict, cache_size: int)[source]#

Evict cache when a new cache comes and the storage is full

Input:

cache_dict: a dict consists of current cache kv_obj: the new kv cache to be injected

Returns:

return a key to be evicted

class lmcache.storage_backend.evictor.LRUEvictor(max_cache_size: float = 10.0)[source]#

Bases: BaseEvictor

LRU cache evictor

update_on_get(key: CacheEngineKey | str, cache_dict: OrderedDict) → None[source]#

Update cache recency when a cache is used

Input:

key: a CacheEngineKey or a str cache_dict: a dict consists of current cache

update_on_put(cache_dict: OrderedDict, cache_size: int)[source]#

Evict cache when a new cache comes and the storage is full

Input:

cache_dict: a dict consists of current cache kv_obj: the new kv cache to be injected

Returns:

a list of keys to be evicted

Return type:

evict_keys