User Tools

Site Tools


python:misc:howto2

Differences

This shows you the differences between two versions of the page.

Link to this comparison view

Both sides previous revision Previous revision
Next revision
Previous revision
python:misc:howto2 [2015/04/19 12:05]
ozan
python:misc:howto2 [2015/05/04 05:50] (current)
Line 1: Line 1:
 +===== Yet Another Python Memoize Module =====
  
 +I want to share my memoize module for python (>=2.7).
 +This module has a decorator function **memoize** for caching the values of functions and methods.
 +Different than other modules available online, this module caches the results of functions within a specified period of time. It clears the cache memory, which is stored as a python dictionary in RAM, when the cache items are invalid after validity period (in seconds).
 +
 +Here is a simple example to show you how to use it
 +<code python>
 +
 +import mymemoize
 +memoize=mymemoize.memoize
 +
 +@memoize(validity_period=60*60)
 +def fun():
 +    return "​Something that takes long time"
 +
 +for i in range(1,​100):​
 +    func()
 +
 +</​code>​
 +
 +Here is the mymemoize module.
 +
 +<file python mymemoize.py>​
 +#​!/​usr/​bin/​python
 +# -*- coding: utf-8 -*-
 +"""​
 +The MIT License (MIT)
 +
 +Copyright (c) 2015 Ozan HACIBEKİROĞLU ([email protected])
 +
 +Permission is hereby granted, free of charge, to any person obtaining a copy
 +of this software and associated documentation files (the "​Software"​),​ to deal
 +in the Software without restriction,​ including without limitation the rights
 +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 +copies of the Software, and to permit persons to whom the Software is
 +furnished to do so, subject to the following conditions:
 +
 +The above copyright notice and this permission notice shall be included in
 +all copies or substantial portions of the Software.
 +
 +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,​
 +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 +THE SOFTWARE.
 +"""​
 +from __future__ import print_function
 +__all__ = ["​memoize"​]
 +import time
 +import threading
 +from random import random
 +from functools import wraps
 +
 +# if multi threading is used, lock mechanism is used
 +RLOCK = threading.RLock()
 +
 +# cache dictionary to store memoize function results
 +_cache = {}
 +
 +# timestamp dictionary to store memoize function results'​ timestamps
 +_timestamps = {}
 +
 +__version__ = "​0.1"​
 +__author__ = u"Ozan HACIBEKİROĞLU"​
 +
 +
 +def _normalize_keys(fid,​ args, kwargs):
 +    """​
 +    function id, function args and kwargs are converted to string and concatenated
 +    to be used as cache dictionary key. This is not a very memory efficient way
 +
 +    :param fid: memoized function unique id
 +    :param args: memoized function args
 +    :param kwargs: memoized function kwargs
 +    :returns: string
 +    """​
 +    return str(fid) + str(args) + str(kwargs)
 +
 +
 +def memoize(key="",​ validity_period=30,​ is_self=0):
 +    """​
 +    Decorator for memoizing the functions and methods
 +
 +    :param key: unique id for the function to be used as cache key 
 +                with __name__ of function attr
 +    :param validity_period:​ number of second cache will be stored at least
 +    :param is_self: must be True if an object'​s method will be memoized else False
 +    """​
 +    def _wrapper(func):​
 +        @wraps(func)
 +        def __wrapper(*args,​ **kwargs):
 +            global _cache
 +            global _timestamps
 +            fid = key + "​."​ + func.__name__
 +            if is_self:
 +                cache_key = _normalize_keys(fid,​ args[1:], kwargs)
 +            else:
 +                cache_key = _normalize_keys(fid,​ args, kwargs)
 +            now = time.time()
 +            if _timestamps.get(cache_key,​ now) > now:
 +#                 ​print("​HIT CACHE:"​+cache_key)
 +                return _cache[cache_key]
 +#             ​print("​NO CACHE:"​+cache_key)
 +            _clean_memoize_cache()
 +            ret = func(*args, **kwargs)
 +            with RLOCK:
 +                _cache[cache_key] = ret
 +                _timestamps[cache_key] = now + validity_period
 +            return ret
 +        return __wrapper
 +    return _wrapper
 +
 +
 +def _clean_memoize_cache():​
 +    """​
 +    Clears invalid cache. Called from memoize decorator.
 +    Probability of clearing invalid caches is 5% of memoize calls
 +    """​
 +    if random() >= 0.05:
 +        return
 +#     ​print("​CACHE CLEAN STARTS"​)
 +    global _timestamps
 +    global _cache
 +    with RLOCK:
 +        now = time.time()
 +        keys = _cache.keys()
 +        copy_timestamps = _timestamps.copy()
 +        for k in keys:
 +            if _timestamps.get(k,​ now) <= now:
 +                del _timestamps[k]
 +                del _cache[k]
 +#                 ​print("​CACHE CLEANED:"​+k)
 +
 +if __name__ == "​__main__":​
 +
 +    @memoize(validity_period=60)
 +    def sample_function(a,​ b, c):
 +        time.sleep(1)
 +        return random()
 +
 +    class SampleClass(object):​
 +        @memoize(key="​SampleClass",​ validity_period=60,​ is_self=1)
 +        def sample_method(self,​ a, b, c):
 +            time.sleep(1)
 +            return random()
 +
 +    s = time.time()
 +    val = sample_function(1,​ 2, 3)
 +    print("​sample_function first call: " +
 +          str(time.time() - s) + " second passed"​)
 +    print("​sample_function return value="​ + str(val))
 +    s = time.time()
 +    val = sample_function(1,​ 2, 3)
 +    print("​sample_function second call: " +
 +          str(time.time() - s) + " second passed"​)
 +    print("​sample_function return value="​ + str(val))
 +    print()
 +
 +    b = SampleClass()
 +    s = time.time()
 +    val = b.sample_method("​a",​ "​b",​ "​c"​)
 +    print("​sample_method first call: " +
 +          str(time.time() - s) + " second passed"​)
 +    print("​sample_method return value="​ + str(val))
 +    s = time.time()
 +    val = b.sample_method("​a",​ "​b",​ "​c"​)
 +    print("​sample_method second call: " +
 +          str(time.time() - s) + " second passed"​)
 +    print("​sample_method return value="​ + str(val))
 +    print()
 +
 +    print("​Cache Dictionary"​)
 +    print(_cache)
 +    print("​Timestamp Dictionary"​)
 +    print(_timestamps)
 +
 +</​file>​
 +{{tag>​python memoize module programming tutorial python_memoize_decorator}}
 +~~DISCUSSION~~
python/misc/howto2.txt · Last modified: 2015/05/04 05:50 (external edit)