I want to share my memoize module for python (>=2.7). This module has a decorator function memoize for caching the values of functions and methods. Different than other modules available online, this module caches the results of functions within a specified period of time. It clears the cache memory, which is stored as a python dictionary in RAM, when the cache items are invalid after validity period (in seconds).
Here is a simple example to show you how to use it
import mymemoize memoize=mymemoize.memoize @memoize(validity_period=60*60) def fun(): return "Something that takes long time" for i in range(1,100): func()
Here is the mymemoize module.
#!/usr/bin/python # -*- coding: utf-8 -*- """ The MIT License (MIT) Copyright (c) 2015 Ozan HACIBEKİROĞLU ([email protected]) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ from __future__ import print_function __all__ = ["memoize"] import time import threading from random import random from functools import wraps # if multi threading is used, lock mechanism is used RLOCK = threading.RLock() # cache dictionary to store memoize function results _cache = {} # timestamp dictionary to store memoize function results' timestamps _timestamps = {} __version__ = "0.1" __author__ = u"Ozan HACIBEKİROĞLU" def _normalize_keys(fid, args, kwargs): """ function id, function args and kwargs are converted to string and concatenated to be used as cache dictionary key. This is not a very memory efficient way :param fid: memoized function unique id :param args: memoized function args :param kwargs: memoized function kwargs :returns: string """ return str(fid) + str(args) + str(kwargs) def memoize(key="", validity_period=30, is_self=0): """ Decorator for memoizing the functions and methods :param key: unique id for the function to be used as cache key with __name__ of function attr :param validity_period: number of second cache will be stored at least :param is_self: must be True if an object's method will be memoized else False """ def _wrapper(func): @wraps(func) def __wrapper(*args, **kwargs): global _cache global _timestamps fid = key + "." + func.__name__ if is_self: cache_key = _normalize_keys(fid, args[1:], kwargs) else: cache_key = _normalize_keys(fid, args, kwargs) now = time.time() if _timestamps.get(cache_key, now) > now: # print("HIT CACHE:"+cache_key) return _cache[cache_key] # print("NO CACHE:"+cache_key) _clean_memoize_cache() ret = func(*args, **kwargs) with RLOCK: _cache[cache_key] = ret _timestamps[cache_key] = now + validity_period return ret return __wrapper return _wrapper def _clean_memoize_cache(): """ Clears invalid cache. Called from memoize decorator. Probability of clearing invalid caches is 5% of memoize calls """ if random() >= 0.05: return # print("CACHE CLEAN STARTS") global _timestamps global _cache with RLOCK: now = time.time() keys = _cache.keys() copy_timestamps = _timestamps.copy() for k in keys: if _timestamps.get(k, now) <= now: del _timestamps[k] del _cache[k] # print("CACHE CLEANED:"+k) if __name__ == "__main__": @memoize(validity_period=60) def sample_function(a, b, c): time.sleep(1) return random() class SampleClass(object): @memoize(key="SampleClass", validity_period=60, is_self=1) def sample_method(self, a, b, c): time.sleep(1) return random() s = time.time() val = sample_function(1, 2, 3) print("sample_function first call: " + str(time.time() - s) + " second passed") print("sample_function return value=" + str(val)) s = time.time() val = sample_function(1, 2, 3) print("sample_function second call: " + str(time.time() - s) + " second passed") print("sample_function return value=" + str(val)) print() b = SampleClass() s = time.time() val = b.sample_method("a", "b", "c") print("sample_method first call: " + str(time.time() - s) + " second passed") print("sample_method return value=" + str(val)) s = time.time() val = b.sample_method("a", "b", "c") print("sample_method second call: " + str(time.time() - s) + " second passed") print("sample_method return value=" + str(val)) print() print("Cache Dictionary") print(_cache) print("Timestamp Dictionary") print(_timestamps)
Discussion