From a6212df9d45b7df157044b1b1720cd997ef0f69f Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Sat, 22 Aug 2020 21:59:29 -0700 Subject: [PATCH 001/126] Remove unused "env" setting from pytest section --- tox.ini | 3 --- 1 file changed, 3 deletions(-) diff --git a/tox.ini b/tox.ini index b2b6d45..8e9c2e0 100644 --- a/tox.ini +++ b/tox.ini @@ -23,9 +23,6 @@ addopts= --ignore tests/plot.py norecursedirs=site-packages testpaths=docs diskcache tests -env = - DJANGO_SETTINGS_MODULE=tests.settings - PYTHONPATH={PWD}:{PWD}/tests [testenv:pylint] deps= From 82b45e8e342ee597dba56f0a42c0d6088b0c2037 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Sat, 22 Aug 2020 21:59:59 -0700 Subject: [PATCH 002/126] Remove nose references --- requirements.txt | 1 - tests/test_core.py | 5 ----- tests/test_fanout.py | 5 ----- 3 files changed, 11 deletions(-) diff --git a/requirements.txt b/requirements.txt index 6c1a277..835f48a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,6 @@ django==2.2.* django_redis doc8 gj -nose pylibmc pylint pytest diff --git a/tests/test_core.py b/tests/test_core.py index 7c38874..36311e5 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1464,8 +1464,3 @@ def fibrec(num): assert hits2 == (hits1 + count) assert misses2 == misses1 - - -if __name__ == '__main__': - import nose - nose.runmodule() diff --git a/tests/test_fanout.py b/tests/test_fanout.py index f5d9b70..390961b 100644 --- a/tests/test_fanout.py +++ b/tests/test_fanout.py @@ -685,8 +685,3 @@ def test_custom_filename_disk(): assert content == str(count) * int(1e5) shutil.rmtree(cache.directory, ignore_errors=True) - - -if __name__ == '__main__': - import nose - nose.runmodule() From 83fe89c80371278d45917b6adde58c179fc5f5e2 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Sat, 22 Aug 2020 22:01:36 -0700 Subject: [PATCH 003/126] Fixes for pylint in Python 3.8 --- diskcache/core.py | 32 ++++++++++++++------------------ diskcache/djangocache.py | 4 ++-- diskcache/persistent.py | 2 +- 3 files changed, 17 insertions(+), 21 deletions(-) diff --git a/diskcache/core.py b/diskcache/core.py index 0c8fd2c..0cf95a1 100644 --- a/diskcache/core.py +++ b/diskcache/core.py @@ -180,7 +180,7 @@ def put(self, key): :return: (database key, raw boolean) pair """ - # pylint: disable=bad-continuation,unidiomatic-typecheck + # pylint: disable=unidiomatic-typecheck type_key = type(key) if type_key is BytesType: @@ -378,17 +378,17 @@ def __init__(self, directory, compress_level=1, **kwargs): """ self.compress_level = compress_level - super(JSONDisk, self).__init__(directory, **kwargs) + super().__init__(directory, **kwargs) def put(self, key): json_bytes = json.dumps(key).encode('utf-8') data = zlib.compress(json_bytes, self.compress_level) - return super(JSONDisk, self).put(data) + return super().put(data) def get(self, key, raw): - data = super(JSONDisk, self).get(key, raw) + data = super().get(key, raw) return json.loads(zlib.decompress(data).decode('utf-8')) @@ -396,11 +396,11 @@ def store(self, value, read, key=UNKNOWN): if not read: json_bytes = json.dumps(value).encode('utf-8') value = zlib.compress(json_bytes, self.compress_level) - return super(JSONDisk, self).store(value, read, key=key) + return super().store(value, read, key=key) def fetch(self, mode, filename, value, read): - data = super(JSONDisk, self).fetch(mode, filename, value, read) + data = super().fetch(mode, filename, value, read) if not read: data = json.loads(zlib.decompress(data).decode('utf-8')) return data @@ -448,7 +448,6 @@ def args_to_key(base, args, kwargs, typed): class Cache(object): "Disk and file backed cache." - # pylint: disable=bad-continuation def __init__(self, directory=None, timeout=60, disk=Disk, **settings): """Initialize cache instance. @@ -461,7 +460,7 @@ def __init__(self, directory=None, timeout=60, disk=Disk, **settings): try: assert issubclass(disk, Disk) except (TypeError, AssertionError): - raise ValueError('disk must subclass diskcache.Disk') + raise ValueError('disk must subclass diskcache.Disk') from None if directory is None: directory = tempfile.mkdtemp(prefix='diskcache-') @@ -482,7 +481,7 @@ def __init__(self, directory=None, timeout=60, disk=Disk, **settings): error.errno, 'Cache directory "%s" does not exist' ' and could not be created' % self._directory - ) + ) from None sql = self._sql_retry @@ -756,7 +755,7 @@ def _transact(self, retry=False, filename=None): continue if filename is not None: _disk_remove(filename) - raise Timeout + raise Timeout from None try: yield sql, filenames.append @@ -1609,8 +1608,7 @@ def pull(self, prefix=None, default=(None, None), side='front', if error.errno == errno.ENOENT: # Key was deleted before we could retrieve result. continue - else: - raise + raise finally: if name is not None: self._disk.remove(name) @@ -1719,8 +1717,7 @@ def peek(self, prefix=None, default=(None, None), side='front', if error.errno == errno.ENOENT: # Key was deleted before we could retrieve result. continue - else: - raise + raise finally: if name is not None: self._disk.remove(name) @@ -1794,8 +1791,7 @@ def peekitem(self, last=True, expire_time=False, tag=False, retry=False): if error.errno == errno.ENOENT: # Key was deleted before we could retrieve result. continue - else: - raise + raise break if expire_time and tag: @@ -2162,7 +2158,7 @@ def cull(self, retry=False): for filename, in rows: cleanup(filename) except Timeout: - raise Timeout(count) + raise Timeout(count) from None return count @@ -2215,7 +2211,7 @@ def _select_delete(self, select, args, row_index=0, arg_index=0, cleanup(row[-1]) except Timeout: - raise Timeout(count) + raise Timeout(count) from None return count diff --git a/diskcache/djangocache.py b/diskcache/djangocache.py index 997b852..329b966 100644 --- a/diskcache/djangocache.py +++ b/diskcache/djangocache.py @@ -22,7 +22,7 @@ def __init__(self, directory, params): :param dict params: cache parameters """ - super(DjangoCache, self).__init__(params) + super().__init__(params) shards = params.get('SHARDS', 8) timeout = params.get('DATABASE_TIMEOUT', 0.010) options = params.get('OPTIONS', {}) @@ -228,7 +228,7 @@ def incr(self, key, delta=1, version=None, default=None, retry=True): try: return self._cache.incr(key, delta, default, retry) except KeyError: - raise ValueError("Key '%s' not found" % key) + raise ValueError("Key '%s' not found" % key) from None def decr(self, key, delta=1, version=None, default=None, retry=True): diff --git a/diskcache/persistent.py b/diskcache/persistent.py index 9de5835..76eb9a2 100644 --- a/diskcache/persistent.py +++ b/diskcache/persistent.py @@ -915,7 +915,7 @@ def popitem(self, last=True): :raises KeyError: if index is empty """ - # pylint: disable=arguments-differ + # pylint: disable=arguments-differ,unbalanced-tuple-unpacking _cache = self._cache with _cache.transact(retry=True): From e197a93e60a10fb8ed6030fc6b556b122a664573 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Sat, 22 Aug 2020 22:25:17 -0700 Subject: [PATCH 004/126] Update 2019 date references to 2020 --- LICENSE | 2 +- README.rst | 4 ++-- docs/conf.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/LICENSE b/LICENSE index 3259b98..d31985f 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright 2016-2019 Grant Jenks +Copyright 2016-2020 Grant Jenks Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the diff --git a/README.rst b/README.rst index 57b7a2e..ba53988 100644 --- a/README.rst +++ b/README.rst @@ -4,7 +4,7 @@ DiskCache: Disk Backed Cache `DiskCache`_ is an Apache2 licensed disk and file backed cache library, written in pure-Python, and compatible with Django. -The cloud-based computing of 2019 puts a premium on memory. Gigabytes of empty +The cloud-based computing of 2020 puts a premium on memory. Gigabytes of empty space is left on disks as processes vie for memory. Among these processes is Memcached (and sometimes Redis) which is used as a cache. Wouldn't it be nice to leverage empty disk space for caching? @@ -388,7 +388,7 @@ Reference License ------- -Copyright 2016-2019 Grant Jenks +Copyright 2016-2020 Grant Jenks Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the diff --git a/docs/conf.py b/docs/conf.py index 683dd3e..2556188 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -51,7 +51,7 @@ # General information about the project. project = u'DiskCache' -copyright = u'2019, Grant Jenks' +copyright = u'2020, Grant Jenks' author = u'Grant Jenks' # The version info for the project you're documenting, acts as replacement for From 2ba5b7985d17141afa7fb5fb3bf5913b0e496fe6 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Sat, 22 Aug 2020 22:44:32 -0700 Subject: [PATCH 005/126] Update references to Python 2 support --- README.rst | 4 ++-- docs/development.rst | 4 +--- setup.py | 5 +---- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/README.rst b/README.rst index ba53988..847b2cf 100644 --- a/README.rst +++ b/README.rst @@ -77,8 +77,8 @@ Features - Thread-safe and process-safe - Supports multiple eviction policies (LRU and LFU included) - Keys support "tag" metadata and eviction -- Developed on Python 3.7 -- Tested on CPython 2.7, 3.4, 3.5, 3.6, 3.7 and PyPy +- Developed on Python 3.8 +- Tested on CPython 3.5, 3.6, 3.7, 3.8 - Tested on Linux, Mac OS X, and Windows - Tested using Travis CI and AppVeyor CI diff --git a/docs/development.rst b/docs/development.rst index 6d16d44..828c3be 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -67,12 +67,10 @@ Testing :doc:`DiskCache ` currently tests against five versions of Python: -* CPython 2.7 -* CPython 3.4 * CPython 3.5 * CPython 3.6 * CPython 3.7 -* PyPy2 +* CPython 3.8 Testing uses `tox `_. If you don't want to install all the development requirements, then, after downloading, you can diff --git a/setup.py b/setup.py index 90dc280..d9be963 100644 --- a/setup.py +++ b/setup.py @@ -38,14 +38,11 @@ def run_tests(self): 'License :: OSI Approved :: Apache Software License', 'Natural Language :: English', 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: Implementation :: CPython', - 'Programming Language :: Python :: Implementation :: PyPy', ), ) From 4497cfc85197d57298120dbd238d263bfb9e9557 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Sat, 22 Aug 2020 22:58:07 -0700 Subject: [PATCH 006/126] Remove Python 2 shims --- diskcache/core.py | 68 +++++++++++------------------------------ diskcache/fanout.py | 29 ++---------------- diskcache/persistent.py | 25 +++------------ diskcache/recipes.py | 17 ++--------- 4 files changed, 27 insertions(+), 112 deletions(-) diff --git a/diskcache/core.py b/diskcache/core.py index 0cf95a1..1ec6acc 100644 --- a/diskcache/core.py +++ b/diskcache/core.py @@ -10,6 +10,7 @@ import json import os import os.path as op +import pickle import pickletools import sqlite3 import struct @@ -20,42 +21,9 @@ import warnings import zlib -############################################################################ -# BEGIN Python 2/3 Shims -############################################################################ - -if sys.hexversion < 0x03000000: - import cPickle as pickle # pylint: disable=import-error - # ISSUE #25 Fix for http://bugs.python.org/issue10211 - from cStringIO import StringIO as BytesIO # pylint: disable=import-error - from thread import get_ident # pylint: disable=import-error,no-name-in-module - TextType = unicode # pylint: disable=invalid-name,undefined-variable - BytesType = str - INT_TYPES = int, long # pylint: disable=undefined-variable - range = xrange # pylint: disable=redefined-builtin,invalid-name,undefined-variable - io_open = io.open # pylint: disable=invalid-name -else: - import pickle - from io import BytesIO # pylint: disable=ungrouped-imports - from threading import get_ident - TextType = str - BytesType = bytes - INT_TYPES = (int,) - io_open = open # pylint: disable=invalid-name - def full_name(func): "Return full name of `func` by adding the module and function name." - try: - # The __qualname__ attribute is only available in Python 3.3 and later. - # GrantJ 2019-03-29 Remove after support for Python 2 is dropped. - name = func.__qualname__ - except AttributeError: - name = func.__name__ - return func.__module__ + '.' + name - -############################################################################ -# END Python 2/3 Shims -############################################################################ + return func.__module__ + '.' + func.__qualname__ try: WindowsError @@ -164,9 +132,9 @@ def hash(self, key): if type_disk_key is sqlite3.Binary: return zlib.adler32(disk_key) & mask - elif type_disk_key is TextType: + elif type_disk_key is str: return zlib.adler32(disk_key.encode('utf-8')) & mask # pylint: disable=no-member - elif type_disk_key in INT_TYPES: + elif type_disk_key is int: return disk_key % mask else: assert type_disk_key is float @@ -183,10 +151,10 @@ def put(self, key): # pylint: disable=unidiomatic-typecheck type_key = type(key) - if type_key is BytesType: + if type_key is bytes: return sqlite3.Binary(key), True - elif ((type_key is TextType) - or (type_key in INT_TYPES + elif ((type_key is str) + or (type_key is int and -9223372036854775808 <= key <= 9223372036854775807) or (type_key is float)): return key, True @@ -206,9 +174,9 @@ def get(self, key, raw): """ # pylint: disable=no-self-use,unidiomatic-typecheck if raw: - return BytesType(key) if type(key) is sqlite3.Binary else key + return bytes(key) if type(key) is sqlite3.Binary else key else: - return pickle.load(BytesIO(key)) + return pickle.load(io.BytesIO(key)) def store(self, value, read, key=UNKNOWN): @@ -225,12 +193,12 @@ def store(self, value, read, key=UNKNOWN): type_value = type(value) min_file_size = self.min_file_size - if ((type_value is TextType and len(value) < min_file_size) - or (type_value in INT_TYPES + if ((type_value is str and len(value) < min_file_size) + or (type_value is int and -9223372036854775808 <= value <= 9223372036854775807) or (type_value is float)): return 0, MODE_RAW, None, value - elif type_value is BytesType: + elif type_value is bytes: if len(value) < min_file_size: return 0, MODE_RAW, None, sqlite3.Binary(value) else: @@ -240,10 +208,10 @@ def store(self, value, read, key=UNKNOWN): writer.write(value) return len(value), MODE_BINARY, filename, None - elif type_value is TextType: + elif type_value is str: filename, full_path = self.filename(key, value) - with io_open(full_path, 'w', encoding='UTF-8') as writer: + with open(full_path, 'w', encoding='UTF-8') as writer: writer.write(value) size = op.getsize(full_path) @@ -286,7 +254,7 @@ def fetch(self, mode, filename, value, read): """ # pylint: disable=no-self-use,unidiomatic-typecheck if mode == MODE_RAW: - return BytesType(value) if type(value) is sqlite3.Binary else value + return bytes(value) if type(value) is sqlite3.Binary else value elif mode == MODE_BINARY: if read: return open(op.join(self._directory, filename), 'rb') @@ -295,14 +263,14 @@ def fetch(self, mode, filename, value, read): return reader.read() elif mode == MODE_TEXT: full_path = op.join(self._directory, filename) - with io_open(full_path, 'r', encoding='UTF-8') as reader: + with open(full_path, 'r', encoding='UTF-8') as reader: return reader.read() elif mode == MODE_PICKLE: if value is None: with open(op.join(self._directory, filename), 'rb') as reader: return pickle.load(reader) else: - return pickle.load(BytesIO(value)) + return pickle.load(io.BytesIO(value)) def filename(self, key=UNKNOWN, value=UNKNOWN): @@ -738,7 +706,7 @@ def _transact(self, retry=False, filename=None): sql = self._sql filenames = [] _disk_remove = self._disk.remove - tid = get_ident() + tid = threading.get_ident() txn_id = self._txn_id if tid == txn_id: diff --git a/diskcache/fanout.py b/diskcache/fanout.py index 8a0a722..1a59aa2 100644 --- a/diskcache/fanout.py +++ b/diskcache/fanout.py @@ -1,5 +1,6 @@ "Fanout cache automatically shards keys and values." +import functools import itertools as it import operator import os.path as op @@ -11,17 +12,6 @@ from .core import ENOVAL, DEFAULT_SETTINGS, Cache, Disk, Timeout from .persistent import Deque, Index -############################################################################ -# BEGIN Python 2/3 Shims -############################################################################ - -if sys.hexversion >= 0x03000000: - from functools import reduce - -############################################################################ -# END Python 2/3 Shims -############################################################################ - class FanoutCache(object): "Cache that shards keys and values." @@ -383,7 +373,7 @@ def check(self, fix=False, retry=False): """ warnings = (shard.check(fix, retry) for shard in self._shards) - return reduce(operator.iadd, warnings, []) + return functools.reduce(operator.iadd, warnings, []) def expire(self, retry=False): @@ -661,17 +651,4 @@ def index(self, name): return temp -############################################################################ -# BEGIN Python 2/3 Shims -############################################################################ - -if sys.hexversion < 0x03000000: - import types - memoize_func = Cache.__dict__['memoize'] # pylint: disable=invalid-name - FanoutCache.memoize = types.MethodType(memoize_func, None, FanoutCache) -else: - FanoutCache.memoize = Cache.memoize - -############################################################################ -# END Python 2/3 Shims -############################################################################ +FanoutCache.memoize = Cache.memoize diff --git a/diskcache/persistent.py b/diskcache/persistent.py index 76eb9a2..5047df5 100644 --- a/diskcache/persistent.py +++ b/diskcache/persistent.py @@ -6,29 +6,12 @@ import sys from collections import OrderedDict +from collections.abc import MutableMapping, Sequence +from collections.abc import KeysView, ValuesView, ItemsView from contextlib import contextmanager from shutil import rmtree -from .core import BytesType, Cache, ENOVAL, TextType - -############################################################################ -# BEGIN Python 2/3 Shims -############################################################################ - -try: - from collections.abc import MutableMapping, Sequence - from collections.abc import KeysView, ValuesView, ItemsView -except ImportError: - from collections import MutableMapping, Sequence - from collections import KeysView, ValuesView, ItemsView - -if sys.hexversion < 0x03000000: - from itertools import izip as zip # pylint: disable=redefined-builtin,no-name-in-module,ungrouped-imports - range = xrange # pylint: disable=redefined-builtin,invalid-name,undefined-variable - -############################################################################ -# END Python 2/3 Shims -############################################################################ +from .core import Cache, ENOVAL def _make_compare(seq_op, doc): @@ -699,7 +682,7 @@ def __init__(self, *args, **kwargs): 4 """ - if args and isinstance(args[0], (BytesType, TextType)): + if args and isinstance(args[0], (bytes, str)): directory = args[0] args = args[1:] else: diff --git a/diskcache/recipes.py b/diskcache/recipes.py index fb64250..010f1f2 100644 --- a/diskcache/recipes.py +++ b/diskcache/recipes.py @@ -12,19 +12,6 @@ from .core import ENOVAL, args_to_key, full_name -############################################################################ -# BEGIN Python 2/3 Shims -############################################################################ - -if sys.hexversion < 0x03000000: - from thread import get_ident # pylint: disable=import-error -else: - from threading import get_ident - -############################################################################ -# END Python 2/3 Shims -############################################################################ - class Averager(object): """Recipe for calculating a running average. @@ -139,7 +126,7 @@ def __init__(self, cache, key, expire=None, tag=None): def acquire(self): "Acquire lock by incrementing count using spin-lock algorithm." pid = os.getpid() - tid = get_ident() + tid = threading.get_ident() pid_tid = '{}-{}'.format(pid, tid) while True: @@ -156,7 +143,7 @@ def acquire(self): def release(self): "Release lock by decrementing count." pid = os.getpid() - tid = get_ident() + tid = threading.get_ident() pid_tid = '{}-{}'.format(pid, tid) with self._cache.transact(retry=True): From 8e0f545a986c8dbb514aaa3c526472e1961934b6 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Sat, 22 Aug 2020 23:13:45 -0700 Subject: [PATCH 007/126] Add flake8 to linters and fix issues --- diskcache/__init__.py | 6 ++++-- diskcache/core.py | 22 +++++++++++++++------- diskcache/fanout.py | 3 +-- diskcache/persistent.py | 5 +++-- diskcache/recipes.py | 7 +++++-- tox.ini | 13 +++++++++++-- 6 files changed, 39 insertions(+), 17 deletions(-) diff --git a/diskcache/__init__.py b/diskcache/__init__.py index 192524e..a301f26 100644 --- a/diskcache/__init__.py +++ b/diskcache/__init__.py @@ -6,7 +6,9 @@ """ -from .core import Cache, Disk, EmptyDirWarning, JSONDisk, UnknownFileWarning, Timeout +from .core import ( + Cache, Disk, EmptyDirWarning, JSONDisk, UnknownFileWarning, Timeout +) from .core import DEFAULT_SETTINGS, ENOVAL, EVICTION_POLICY, UNKNOWN from .fanout import FanoutCache from .persistent import Deque, Index @@ -37,7 +39,7 @@ ] try: - from .djangocache import DjangoCache # pylint: disable=wrong-import-position + from .djangocache import DjangoCache # noqa __all__.append('DjangoCache') except Exception: # pylint: disable=broad-except # Django not installed or not setup so ignore. diff --git a/diskcache/core.py b/diskcache/core.py index 1ec6acc..4317160 100644 --- a/diskcache/core.py +++ b/diskcache/core.py @@ -14,23 +14,25 @@ import pickletools import sqlite3 import struct -import sys import tempfile import threading import time import warnings import zlib + def full_name(func): "Return full name of `func` by adding the module and function name." return func.__module__ + '.' + func.__qualname__ + try: WindowsError except NameError: class WindowsError(Exception): "Windows error place-holder on platforms without support." + class Constant(tuple): "Pretty display of immutable constant." def __new__(cls, name): @@ -39,6 +41,7 @@ def __new__(cls, name): def __repr__(self): return '%s' % self[0] + DBNAME = 'cache.db' ENOVAL = Constant('ENOVAL') UNKNOWN = Constant('UNKNOWN') @@ -133,7 +136,7 @@ def hash(self, key): if type_disk_key is sqlite3.Binary: return zlib.adler32(disk_key) & mask elif type_disk_key is str: - return zlib.adler32(disk_key.encode('utf-8')) & mask # pylint: disable=no-member + return zlib.adler32(disk_key.encode('utf-8')) & mask # noqa elif type_disk_key is int: return disk_key % mask else: @@ -1056,7 +1059,9 @@ def incr(self, key, delta=1, default=0, retry=False): raise KeyError(key) value = default + delta - columns = (None, None) + self._disk.store(value, False, key=key) + columns = ( + (None, None) + self._disk.store(value, False, key=key) + ) self._row_insert(db_key, raw, now, columns) self._cull(now, sql, cleanup) return value @@ -1068,7 +1073,9 @@ def incr(self, key, delta=1, default=0, retry=False): raise KeyError(key) value = default + delta - columns = (None, None) + self._disk.store(value, False, key=key) + columns = ( + (None, None) + self._disk.store(value, False, key=key) + ) self._row_update(rowid, now, columns) self._cull(now, sql, cleanup) cleanup(filename) @@ -1184,7 +1191,7 @@ def get(self, key, default=None, read=False, expire_time=False, tag=False, return default (rowid, db_expire_time, db_tag, - mode, filename, db_value), = rows + mode, filename, db_value), = rows # noqa: E127 try: value = self._disk.fetch(mode, filename, db_value, read) @@ -1269,7 +1276,7 @@ def __contains__(self, key): return bool(rows) - def pop(self, key, default=None, expire_time=False, tag=False, retry=False): + def pop(self, key, default=None, expire_time=False, tag=False, retry=False): # noqa: E501 """Remove corresponding item for `key` from cache and return value. If `key` is missing, return `default`. @@ -2341,7 +2348,8 @@ def close(self): def __enter__(self): # Create connection in thread. - connection = self._con # pylint: disable=unused-variable + # pylint: disable=unused-variable + connection = self._con # noqa return self diff --git a/diskcache/fanout.py b/diskcache/fanout.py index 1a59aa2..a579a17 100644 --- a/diskcache/fanout.py +++ b/diskcache/fanout.py @@ -5,7 +5,6 @@ import operator import os.path as op import sqlite3 -import sys import tempfile import time @@ -290,7 +289,7 @@ def __contains__(self, key): return key in shard - def pop(self, key, default=None, expire_time=False, tag=False, retry=False): + def pop(self, key, default=None, expire_time=False, tag=False, retry=False): # noqa: E501 """Remove corresponding item for `key` from cache and return value. If `key` is missing, return `default`. diff --git a/diskcache/persistent.py b/diskcache/persistent.py index 5047df5..2e40074 100644 --- a/diskcache/persistent.py +++ b/diskcache/persistent.py @@ -197,8 +197,9 @@ def __setitem__(self, index, value): :raises IndexError: if index out of range """ - set_value = lambda key: self._cache.__setitem__(key, value) - self._index(index, set_value) + def _set_value(key): + return self._cache.__setitem__(key, value) + self._index(index, _set_value) def __delitem__(self, index): diff --git a/diskcache/recipes.py b/diskcache/recipes.py index 010f1f2..1a5890d 100644 --- a/diskcache/recipes.py +++ b/diskcache/recipes.py @@ -6,7 +6,6 @@ import math import os import random -import sys import threading import time @@ -82,7 +81,11 @@ def acquire(self): "Acquire lock using spin-lock algorithm." while True: added = self._cache.add( - self._key, None, expire=self._expire, tag=self._tag, retry=True, + self._key, + None, + expire=self._expire, + tag=self._tag, + retry=True, ) if added: break diff --git a/tox.ini b/tox.ini index 8e9c2e0..b54e574 100644 --- a/tox.ini +++ b/tox.ini @@ -9,7 +9,7 @@ deps= pytest-django pytest-xdist commands=python -m pytest -setenv = +setenv= DJANGO_SETTINGS_MODULE=tests.settings PYTHONPATH={toxinidir} @@ -27,8 +27,17 @@ testpaths=docs diskcache tests [testenv:pylint] deps= django==2.2.* + flake8 pylint -commands=pylint diskcache +commands= + flake8 diskcache + pylint diskcache [doc8] ignore=D000 + +[flake8] +ignore= + E124 + E303 + W503 From 4f2ff540155c0182ff07a5a3408aceb8c10cff1e Mon Sep 17 00:00:00 2001 From: ume Date: Sat, 1 Aug 2020 10:26:23 +0900 Subject: [PATCH 008/126] Add locked method to Lock --- diskcache/recipes.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/diskcache/recipes.py b/diskcache/recipes.py index 1a5890d..85509c7 100644 --- a/diskcache/recipes.py +++ b/diskcache/recipes.py @@ -95,6 +95,10 @@ def release(self): "Release lock by deleting key." self._cache.delete(self._key, retry=True) + def locked(self): + "Return true if the lock is acquired." + return self._key in self._cache + def __enter__(self): self.acquire() From a8a014cb96ed0d96c12d07b6b7d4729b54c0c011 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Sun, 23 Aug 2020 15:47:49 -0700 Subject: [PATCH 009/126] Add paragraph to caveats about cache volume for Issue #140 --- docs/tutorial.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index b29322c..8f8dc0f 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -892,6 +892,12 @@ thread-pool executor asynchronously. For example:: asyncio.run(set_async('test-key', 'test-value')) +The cache :meth:`volume ` is based on the size of the +database that stores metadata and the size of the values stored in files. It +does not account the size of directories themselves or other filesystem +metadata. If directory count or size is a concern then consider implementing an +alternative :class:`Disk `. + .. _`hash protocol`: https://docs.python.org/library/functions.html#hash .. _`not recommended`: https://www.sqlite.org/faq.html#q5 .. _`performs poorly`: https://www.pythonanywhere.com/forums/topic/1847/ From b1ab12c7a85bf5473fdc491d2d03c9adc6de5bba Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Sun, 23 Aug 2020 15:56:19 -0700 Subject: [PATCH 010/126] Bump version to 5.0.0 --- diskcache/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/diskcache/__init__.py b/diskcache/__init__.py index a301f26..62766c5 100644 --- a/diskcache/__init__.py +++ b/diskcache/__init__.py @@ -46,8 +46,8 @@ pass __title__ = 'diskcache' -__version__ = '4.1.0' -__build__ = 0x040100 +__version__ = '5.0.0' +__build__ = 0x050000 __author__ = 'Grant Jenks' __license__ = 'Apache 2.0' -__copyright__ = 'Copyright 2016-2018 Grant Jenks' +__copyright__ = 'Copyright 2016-2020 Grant Jenks' From 38bd9019e1b8d5adec52f7bc893a6f662c49cd20 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Sun, 23 Aug 2020 16:23:02 -0700 Subject: [PATCH 011/126] Remove Python 2 mapping methods keys/values/items/iter*/view* --- diskcache/persistent.py | 207 ++++++---------------------------------- 1 file changed, 30 insertions(+), 177 deletions(-) diff --git a/diskcache/persistent.py b/diskcache/persistent.py index 2e40074..4324660 100644 --- a/diskcache/persistent.py +++ b/diskcache/persistent.py @@ -1031,196 +1031,49 @@ def __len__(self): return len(self._cache) - if sys.hexversion < 0x03000000: - def keys(self): - """List of index keys. + def keys(self): + """Set-like object providing a view of index keys. - >>> index = Index() - >>> index.update([('a', 1), ('b', 2), ('c', 3)]) - >>> index.keys() - ['a', 'b', 'c'] - - :return: list of keys - - """ - return list(self._cache) - - - def values(self): - """List of index values. - - >>> index = Index() - >>> index.update([('a', 1), ('b', 2), ('c', 3)]) - >>> index.values() - [1, 2, 3] - - :return: list of values - - """ - return list(self.itervalues()) - - - def items(self): - """List of index items. - - >>> index = Index() - >>> index.update([('a', 1), ('b', 2), ('c', 3)]) - >>> index.items() - [('a', 1), ('b', 2), ('c', 3)] - - :return: list of items - - """ - return list(self.iteritems()) - - - def iterkeys(self): - """Iterator of index keys. - - >>> index = Index() - >>> index.update([('a', 1), ('b', 2), ('c', 3)]) - >>> list(index.iterkeys()) - ['a', 'b', 'c'] - - :return: iterator of keys - - """ - return iter(self._cache) - - - def itervalues(self): - """Iterator of index values. - - >>> index = Index() - >>> index.update([('a', 1), ('b', 2), ('c', 3)]) - >>> list(index.itervalues()) - [1, 2, 3] - - :return: iterator of values - - """ - _cache = self._cache - - for key in _cache: - while True: - try: - yield _cache[key] - except KeyError: - pass - break - - - def iteritems(self): - """Iterator of index items. - - >>> index = Index() - >>> index.update([('a', 1), ('b', 2), ('c', 3)]) - >>> list(index.iteritems()) - [('a', 1), ('b', 2), ('c', 3)] - - :return: iterator of items - - """ - _cache = self._cache - - for key in _cache: - while True: - try: - yield key, _cache[key] - except KeyError: - pass - break - - - def viewkeys(self): - """Set-like object providing a view of index keys. - - >>> index = Index() - >>> index.update({'a': 1, 'b': 2, 'c': 3}) - >>> keys_view = index.viewkeys() - >>> 'b' in keys_view - True - - :return: keys view - - """ - return KeysView(self) - - - def viewvalues(self): - """Set-like object providing a view of index values. - - >>> index = Index() - >>> index.update({'a': 1, 'b': 2, 'c': 3}) - >>> values_view = index.viewvalues() - >>> 2 in values_view - True - - :return: values view - - """ - return ValuesView(self) - - - def viewitems(self): - """Set-like object providing a view of index items. - - >>> index = Index() - >>> index.update({'a': 1, 'b': 2, 'c': 3}) - >>> items_view = index.viewitems() - >>> ('b', 2) in items_view - True - - :return: items view - - """ - return ItemsView(self) - - - else: - def keys(self): - """Set-like object providing a view of index keys. - - >>> index = Index() - >>> index.update({'a': 1, 'b': 2, 'c': 3}) - >>> keys_view = index.keys() - >>> 'b' in keys_view - True + >>> index = Index() + >>> index.update({'a': 1, 'b': 2, 'c': 3}) + >>> keys_view = index.keys() + >>> 'b' in keys_view + True - :return: keys view + :return: keys view - """ - return KeysView(self) + """ + return KeysView(self) - def values(self): - """Set-like object providing a view of index values. + def values(self): + """Set-like object providing a view of index values. - >>> index = Index() - >>> index.update({'a': 1, 'b': 2, 'c': 3}) - >>> values_view = index.values() - >>> 2 in values_view - True + >>> index = Index() + >>> index.update({'a': 1, 'b': 2, 'c': 3}) + >>> values_view = index.values() + >>> 2 in values_view + True - :return: values view + :return: values view - """ - return ValuesView(self) + """ + return ValuesView(self) - def items(self): - """Set-like object providing a view of index items. + def items(self): + """Set-like object providing a view of index items. - >>> index = Index() - >>> index.update({'a': 1, 'b': 2, 'c': 3}) - >>> items_view = index.items() - >>> ('b', 2) in items_view - True + >>> index = Index() + >>> index.update({'a': 1, 'b': 2, 'c': 3}) + >>> items_view = index.items() + >>> ('b', 2) in items_view + True - :return: items view + :return: items view - """ - return ItemsView(self) + """ + return ItemsView(self) __hash__ = None From a1bfadd2c72304cea6aa324a0b40b07b35568ed8 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Sun, 23 Aug 2020 16:23:36 -0700 Subject: [PATCH 012/126] Update tests for Python 3 --- tests/benchmark_core.py | 9 +-------- tests/benchmark_djangocache.py | 9 +-------- tests/benchmark_glob.py | 2 -- tests/benchmark_incr.py | 2 -- tests/stress_test_core.py | 17 +++-------------- tests/stress_test_deque.py | 2 -- tests/stress_test_deque_mp.py | 2 -- tests/stress_test_fanout.py | 17 +++-------------- tests/stress_test_index.py | 2 -- tests/stress_test_index_mp.py | 2 -- tests/test_core.py | 6 +----- tests/test_fanout.py | 5 ----- tests/utils.py | 2 -- 13 files changed, 9 insertions(+), 68 deletions(-) diff --git a/tests/benchmark_core.py b/tests/benchmark_core.py index 1811de0..2eeae3b 100644 --- a/tests/benchmark_core.py +++ b/tests/benchmark_core.py @@ -6,23 +6,16 @@ """ -from __future__ import print_function - import collections as co import multiprocessing as mp import os +import pickle import random import shutil import sys import time import warnings -if sys.hexversion < 0x03000000: - range = xrange - import cPickle as pickle -else: - import pickle - from utils import display PROCS = 8 diff --git a/tests/benchmark_djangocache.py b/tests/benchmark_djangocache.py index 898188f..0512752 100644 --- a/tests/benchmark_djangocache.py +++ b/tests/benchmark_djangocache.py @@ -6,23 +6,16 @@ """ -from __future__ import print_function - import collections as co import multiprocessing as mp import os +import pickle import random import shutil import sys import time import warnings -if sys.hexversion < 0x03000000: - range = xrange - import cPickle as pickle -else: - import pickle - from utils import display PROCS = 8 diff --git a/tests/benchmark_glob.py b/tests/benchmark_glob.py index 0402ef8..82237de 100644 --- a/tests/benchmark_glob.py +++ b/tests/benchmark_glob.py @@ -1,7 +1,5 @@ "Benchmark glob.glob1 as used by django.core.cache.backends.filebased." -from __future__ import print_function - import os import os.path as op import shutil diff --git a/tests/benchmark_incr.py b/tests/benchmark_incr.py index 4a01628..9c8e2fa 100644 --- a/tests/benchmark_incr.py +++ b/tests/benchmark_incr.py @@ -2,8 +2,6 @@ """ -from __future__ import print_function - import json import multiprocessing as mp import shutil diff --git a/tests/stress_test_core.py b/tests/stress_test_core.py index bce0d16..68f2ebd 100644 --- a/tests/stress_test_core.py +++ b/tests/stress_test_core.py @@ -1,11 +1,11 @@ "Stress test diskcache.core.Cache." -from __future__ import print_function - import collections as co from diskcache import Cache, UnknownFileWarning, EmptyDirWarning, Timeout import multiprocessing as mp import os +import pickle +import queue import random import shutil import sys @@ -13,17 +13,6 @@ import time import warnings -try: - import Queue -except ImportError: - import queue as Queue - -if sys.hexversion < 0x03000000: - range = xrange - import cPickle as pickle -else: - import pickle - from .utils import display OPERATIONS = int(1e4) @@ -163,7 +152,7 @@ def dispatch(num, eviction_policy, processes, threads): with open('input-%s.pkl' % num, 'rb') as reader: process_queue = pickle.load(reader) - thread_queues = [Queue.Queue() for _ in range(threads)] + thread_queues = [queue.Queue() for _ in range(threads)] subthreads = [ threading.Thread( target=worker, args=(thread_queue, eviction_policy, processes, threads) diff --git a/tests/stress_test_deque.py b/tests/stress_test_deque.py index cf48812..7b3ac2f 100644 --- a/tests/stress_test_deque.py +++ b/tests/stress_test_deque.py @@ -1,7 +1,5 @@ """Stress test diskcache.persistent.Deque.""" -from __future__ import print_function - import collections as co import functools as ft import itertools as it diff --git a/tests/stress_test_deque_mp.py b/tests/stress_test_deque_mp.py index 4624d71..db7b5c4 100644 --- a/tests/stress_test_deque_mp.py +++ b/tests/stress_test_deque_mp.py @@ -1,7 +1,5 @@ """Stress test diskcache.persistent.Deque.""" -from __future__ import print_function - import functools as ft import itertools as it import multiprocessing as mp diff --git a/tests/stress_test_fanout.py b/tests/stress_test_fanout.py index 080b8d8..422e874 100644 --- a/tests/stress_test_fanout.py +++ b/tests/stress_test_fanout.py @@ -1,11 +1,11 @@ "Stress test diskcache.core.Cache." -from __future__ import print_function - import collections as co from diskcache import FanoutCache, UnknownFileWarning, EmptyDirWarning import multiprocessing as mp import os +import pickle +import queue import random import shutil import sys @@ -13,17 +13,6 @@ import time import warnings -try: - import Queue -except ImportError: - import queue as Queue - -if sys.hexversion < 0x03000000: - range = xrange - import cPickle as pickle -else: - import pickle - from .utils import display OPERATIONS = int(1e4) @@ -155,7 +144,7 @@ def dispatch(num, eviction_policy, processes, threads): with open('input-%s.pkl' % num, 'rb') as reader: process_queue = pickle.load(reader) - thread_queues = [Queue.Queue() for _ in range(threads)] + thread_queues = [queue.Queue() for _ in range(threads)] subthreads = [ threading.Thread( target=worker, args=(thread_queue, eviction_policy, processes, threads) diff --git a/tests/stress_test_index.py b/tests/stress_test_index.py index 2846d9c..e7ba3f6 100644 --- a/tests/stress_test_index.py +++ b/tests/stress_test_index.py @@ -1,7 +1,5 @@ """Stress test diskcache.persistent.Index.""" -from __future__ import print_function - import collections as co import itertools as it import random diff --git a/tests/stress_test_index_mp.py b/tests/stress_test_index_mp.py index b3ed813..f8718f0 100644 --- a/tests/stress_test_index_mp.py +++ b/tests/stress_test_index_mp.py @@ -1,7 +1,5 @@ """Stress test diskcache.persistent.Index.""" -from __future__ import print_function - import itertools as it import multiprocessing as mp import os diff --git a/tests/test_core.py b/tests/test_core.py index 36311e5..31b4034 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -22,14 +22,10 @@ from unittest import mock -import diskcache import diskcache as dc pytestmark = pytest.mark.filterwarnings('ignore', category=dc.EmptyDirWarning) -if sys.hexversion < 0x03000000: - range = xrange - @pytest.fixture def cache(): with dc.Cache() as cache: @@ -100,7 +96,7 @@ def test_custom_disk(): shutil.rmtree(cache.directory, ignore_errors=True) -class SHA256FilenameDisk(diskcache.Disk): +class SHA256FilenameDisk(dc.Disk): def filename(self, key=dc.UNKNOWN, value=dc.UNKNOWN): filename = hashlib.sha256(key).hexdigest()[:32] full_path = op.join(self._directory, filename) diff --git a/tests/test_fanout.py b/tests/test_fanout.py index 390961b..aa3c833 100644 --- a/tests/test_fanout.py +++ b/tests/test_fanout.py @@ -1,7 +1,5 @@ "Test diskcache.fanout.FanoutCache." -from __future__ import print_function - import collections as co import errno import functools as ft @@ -28,9 +26,6 @@ warnings.simplefilter('error') warnings.simplefilter('ignore', category=dc.EmptyDirWarning) -if sys.hexversion < 0x03000000: - range = xrange - @pytest.fixture def cache(): diff --git a/tests/utils.py b/tests/utils.py index f2370da..47da791 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,5 +1,3 @@ -from __future__ import print_function - import os import subprocess as sp From cdeee61aa15335fd95ef923f6807cc451d05209b Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Sun, 23 Aug 2020 16:30:49 -0700 Subject: [PATCH 013/126] Bump version to 5.0.1 --- diskcache/__init__.py | 4 ++-- diskcache/persistent.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/diskcache/__init__.py b/diskcache/__init__.py index 62766c5..e1eb93f 100644 --- a/diskcache/__init__.py +++ b/diskcache/__init__.py @@ -46,8 +46,8 @@ pass __title__ = 'diskcache' -__version__ = '5.0.0' -__build__ = 0x050000 +__version__ = '5.0.1' +__build__ = 0x050001 __author__ = 'Grant Jenks' __license__ = 'Apache 2.0' __copyright__ = 'Copyright 2016-2020 Grant Jenks' diff --git a/diskcache/persistent.py b/diskcache/persistent.py index 4324660..d0a452e 100644 --- a/diskcache/persistent.py +++ b/diskcache/persistent.py @@ -3,7 +3,6 @@ """ import operator as op -import sys from collections import OrderedDict from collections.abc import MutableMapping, Sequence From 1f339c12fec3baf6955cd6e1596a16bb6bba447b Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Tue, 25 Aug 2020 21:54:32 -0700 Subject: [PATCH 014/126] Bump version to 5.0.2 --- diskcache/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/diskcache/__init__.py b/diskcache/__init__.py index e1eb93f..01efa1b 100644 --- a/diskcache/__init__.py +++ b/diskcache/__init__.py @@ -46,8 +46,8 @@ pass __title__ = 'diskcache' -__version__ = '5.0.1' -__build__ = 0x050001 +__version__ = '5.0.2' +__build__ = 0x050002 __author__ = 'Grant Jenks' __license__ = 'Apache 2.0' __copyright__ = 'Copyright 2016-2020 Grant Jenks' From d6c3a4950beeca124575a5b6d039af381ec00ba0 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Tue, 8 Sep 2020 21:45:45 -0700 Subject: [PATCH 015/126] Add python_requires kwarg to setup --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index d9be963..f8c465e 100644 --- a/setup.py +++ b/setup.py @@ -31,6 +31,7 @@ def run_tests(self): packages=['diskcache'], tests_require=['tox'], cmdclass={'test': Tox}, + python_requires='>=3', install_requires=[], classifiers=( 'Development Status :: 5 - Production/Stable', From 9670fbb957af8caac826ddc1144da913682de447 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Tue, 8 Sep 2020 21:46:12 -0700 Subject: [PATCH 016/126] Bump version to 5.0.3 --- diskcache/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/diskcache/__init__.py b/diskcache/__init__.py index 01efa1b..a98cfee 100644 --- a/diskcache/__init__.py +++ b/diskcache/__init__.py @@ -46,8 +46,8 @@ pass __title__ = 'diskcache' -__version__ = '5.0.2' -__build__ = 0x050002 +__version__ = '5.0.3' +__build__ = 0x050003 __author__ = 'Grant Jenks' __license__ = 'Apache 2.0' __copyright__ = 'Copyright 2016-2020 Grant Jenks' From 49205f5fc25bb0886e9dcc764ec2dcc3d8afe36d Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Sun, 8 Nov 2020 19:25:54 -0800 Subject: [PATCH 017/126] Prevent cache shard attribute access when unsafe --- diskcache/fanout.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/diskcache/fanout.py b/diskcache/fanout.py index a579a17..3b8ae30 100644 --- a/diskcache/fanout.py +++ b/diskcache/fanout.py @@ -58,6 +58,9 @@ def directory(self): def __getattr__(self, name): + safe_names = {'timeout', 'disk'} + valid_name = name in DEFAULT_SETTINGS or name in safe_names + assert valid_name, f'cannot access {name} in cache shard' return getattr(self._shards[0], name) From 77ca254ae2fda78550ffd678c7666d62ea0c0af1 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Sun, 8 Nov 2020 19:26:11 -0800 Subject: [PATCH 018/126] Support transactions in FanoutCache (probably a bad idea) --- diskcache/fanout.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/diskcache/fanout.py b/diskcache/fanout.py index 3b8ae30..7b85df5 100644 --- a/diskcache/fanout.py +++ b/diskcache/fanout.py @@ -1,5 +1,6 @@ "Fanout cache automatically shards keys and values." +import contextlib as cl import functools import itertools as it import operator @@ -64,6 +65,40 @@ def __getattr__(self, name): return getattr(self._shards[0], name) + @cl.contextmanager + def transact(self, retry=True): + """Context manager to perform a transaction by locking the cache. + + While the cache is locked, no other write operation is permitted. + Transactions should therefore be as short as possible. Read and write + operations performed in a transaction are atomic. Read operations may + occur concurrent to a transaction. + + Transactions may be nested and may not be shared between threads. + + Blocks until transactions are held on all cache shards by retrying as + necessary. + + >>> cache = FanoutCache() + >>> with cache.transact(): # Atomically increment two keys. + ... _ = cache.incr('total', 123.4) + ... _ = cache.incr('count', 1) + >>> with cache.transact(): # Atomically calculate average. + ... average = cache['total'] / cache['count'] + >>> average + 123.4 + + :return: context manager for use in `with` statement + + """ + assert retry, 'retry must be True in FanoutCache' + with cl.ExitStack() as stack: + for shard in self._shards: + shard_transaction = shard.transact(retry=True) + stack.enter_context(shard_transaction) + yield + + def set(self, key, value, expire=None, read=False, tag=None, retry=False): """Set `key` and `value` item in cache. From c5259bd6b6046789ae2bcdf1955e1781adb07bd4 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Sun, 8 Nov 2020 19:27:27 -0800 Subject: [PATCH 019/126] Bump version to 5.1.0 --- diskcache/__init__.py | 4 ++-- diskcache/fanout.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/diskcache/__init__.py b/diskcache/__init__.py index a98cfee..e4d747b 100644 --- a/diskcache/__init__.py +++ b/diskcache/__init__.py @@ -46,8 +46,8 @@ pass __title__ = 'diskcache' -__version__ = '5.0.3' -__build__ = 0x050003 +__version__ = '5.1.0' +__build__ = 0x050100 __author__ = 'Grant Jenks' __license__ = 'Apache 2.0' __copyright__ = 'Copyright 2016-2020 Grant Jenks' diff --git a/diskcache/fanout.py b/diskcache/fanout.py index 7b85df5..7e227c8 100644 --- a/diskcache/fanout.py +++ b/diskcache/fanout.py @@ -61,7 +61,7 @@ def directory(self): def __getattr__(self, name): safe_names = {'timeout', 'disk'} valid_name = name in DEFAULT_SETTINGS or name in safe_names - assert valid_name, f'cannot access {name} in cache shard' + assert valid_name, 'cannot access {} in cache shard'.format(name) return getattr(self._shards[0], name) From 40ce0dedb90ccefacaea41ac11c19ddd491d5a6d Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Sun, 8 Nov 2020 20:24:02 -0800 Subject: [PATCH 020/126] Use no hardcoded /tmp/diskcache/... paths in tests --- tests/test_deque.py | 5 +++-- tests/test_index.py | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_deque.py b/tests/test_deque.py index ddf2338..1ca966d 100644 --- a/tests/test_deque.py +++ b/tests/test_deque.py @@ -4,6 +4,7 @@ import pickle import pytest import shutil +import tempfile from unittest import mock @@ -26,7 +27,7 @@ def deque(): def test_init(): - directory = '/tmp/diskcache/deque' + directory = tempfile.mkdtemp() sequence = list('abcde') deque = dc.Deque(sequence, None) @@ -156,7 +157,7 @@ def test_indexerror(deque): def test_repr(): - directory = '/tmp/diskcache/deque' + directory = tempfile.mkdtemp() deque = dc.Deque(directory=directory) assert repr(deque) == 'Deque(directory=%r)' % directory diff --git a/tests/test_index.py b/tests/test_index.py index ef2a0d0..575558c 100644 --- a/tests/test_index.py +++ b/tests/test_index.py @@ -5,6 +5,7 @@ import pytest import shutil import sys +import tempfile from unittest import mock @@ -26,7 +27,7 @@ def index(): def test_init(): - directory = '/tmp/diskcache/index' + directory = tempfile.mkdtemp() mapping = {'a': 5, 'b': 4, 'c': 3, 'd': 2, 'e': 1} index = dc.Index(None, mapping) From c4ba1f78bb8494bcf6aba9d7d1c3aa49a1093508 Mon Sep 17 00:00:00 2001 From: Cologler Date: Sat, 12 Dec 2020 00:36:01 +0800 Subject: [PATCH 021/126] replace open mode 'w' to 'x' --- diskcache/core.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/diskcache/core.py b/diskcache/core.py index 4317160..1fae11b 100644 --- a/diskcache/core.py +++ b/diskcache/core.py @@ -207,14 +207,14 @@ def store(self, value, read, key=UNKNOWN): else: filename, full_path = self.filename(key, value) - with open(full_path, 'wb') as writer: + with open(full_path, 'xb') as writer: writer.write(value) return len(value), MODE_BINARY, filename, None elif type_value is str: filename, full_path = self.filename(key, value) - with open(full_path, 'w', encoding='UTF-8') as writer: + with open(full_path, 'x', encoding='UTF-8') as writer: writer.write(value) size = op.getsize(full_path) @@ -224,7 +224,7 @@ def store(self, value, read, key=UNKNOWN): reader = ft.partial(value.read, 2 ** 22) filename, full_path = self.filename(key, value) - with open(full_path, 'wb') as writer: + with open(full_path, 'xb') as writer: for chunk in iter(reader, b''): size += len(chunk) writer.write(chunk) @@ -238,7 +238,7 @@ def store(self, value, read, key=UNKNOWN): else: filename, full_path = self.filename(key, value) - with open(full_path, 'wb') as writer: + with open(full_path, 'xb') as writer: writer.write(result) return len(result), MODE_PICKLE, filename, None From ce44a42cfe01ac48e167efeff2612a6e768fdf55 Mon Sep 17 00:00:00 2001 From: C2D <50617709+i404788@users.noreply.github.com> Date: Wed, 20 Jan 2021 16:34:02 +0000 Subject: [PATCH 022/126] Use disk provided by the user whenever possible --- diskcache/fanout.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/diskcache/fanout.py b/diskcache/fanout.py index 7e227c8..b39c744 100644 --- a/diskcache/fanout.py +++ b/diskcache/fanout.py @@ -36,6 +36,7 @@ def __init__(self, directory=None, shards=8, timeout=0.010, disk=Disk, self._count = shards self._directory = directory + self._disk = disk self._shards = tuple( Cache( directory=op.join(directory, '%03d' % num), @@ -622,7 +623,7 @@ def cache(self, name): except KeyError: parts = name.split('/') directory = op.join(self._directory, 'cache', *parts) - temp = Cache(directory=directory) + temp = Cache(directory=directory, disk=self._disk) _caches[name] = temp return temp From 9a300bd4ab4896cfc4eb90703d155d84924be447 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Wed, 20 Jan 2021 20:27:27 -0800 Subject: [PATCH 023/126] Remove transaction from Deque.__init__ When initializing a Deque, a transaction was used to extend elements from the given iterable. The transaction is not used in Index.__init__ or in the FanoutCache.fromcache API. Users that want Deque.__init__ to use a transaction as before should use: d = Deque() with d.transact(): d.extend(iterable) The transaction is therefore explicit and consistent with other APIs. --- diskcache/persistent.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/diskcache/persistent.py b/diskcache/persistent.py index d0a452e..6fc072b 100644 --- a/diskcache/persistent.py +++ b/diskcache/persistent.py @@ -81,8 +81,7 @@ def __init__(self, iterable=(), directory=None): """ self._cache = Cache(directory, eviction_policy='none') - with self.transact(): - self.extend(iterable) + self.extend(iterable) @classmethod From 8b0f68b8c208ddd5a6c06e5c6a9d55ef37f5fb1a Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Wed, 20 Jan 2021 20:37:55 -0800 Subject: [PATCH 024/126] Use the same Disk in FanoutCache as in Index and Deque subdirs --- diskcache/fanout.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/diskcache/fanout.py b/diskcache/fanout.py index b39c744..9870c06 100644 --- a/diskcache/fanout.py +++ b/diskcache/fanout.py @@ -652,9 +652,10 @@ def deque(self, name): except KeyError: parts = name.split('/') directory = op.join(self._directory, 'deque', *parts) - temp = Deque(directory=directory) - _deques[name] = temp - return temp + cache = Cache(directory=directory, disk=self._disk) + deque = Deque.fromcache(cache) + _deques[name] = deque + return deque def index(self, name): @@ -684,9 +685,10 @@ def index(self, name): except KeyError: parts = name.split('/') directory = op.join(self._directory, 'index', *parts) - temp = Index(directory) - _indexes[name] = temp - return temp + cache = Cache(directory=directory, disk=self._disk) + index = Index.fromcache(cache) + _indexes[name] = index + return index FanoutCache.memoize = Cache.memoize From fcd31646cf81b6762d1d31cffafb8fd97510f024 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Fri, 22 Jan 2021 10:29:07 -0800 Subject: [PATCH 025/126] Remove travis and appveyor in favor of GitHub Actions --- .github/workflows/integration.yml | 52 ++++++++++++++++++ .github/workflows/release.yml | 38 +++++++++++++ .travis.yml | 19 ------- appveyor.yml | 22 -------- tox.ini | 88 ++++++++++++++++++++++++------- 5 files changed, 159 insertions(+), 60 deletions(-) create mode 100644 .github/workflows/integration.yml create mode 100644 .github/workflows/release.yml delete mode 100644 .travis.yml delete mode 100644 appveyor.yml diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml new file mode 100644 index 0000000..7dfc198 --- /dev/null +++ b/.github/workflows/integration.yml @@ -0,0 +1,52 @@ +name: integration + +on: [push] + +jobs: + + checks: + runs-on: ubuntu-latest + strategy: + max-parallel: 8 + matrix: + check: [bluecheck, doc8, docs, flake8, isortcheck, mypy, pylint, rstcheck] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.9 + - name: Install dependencies + run: | + pip install --upgrade pip + pip install tox + - name: Run checks with tox + run: | + tox -e ${{ matrix.check }} + + tests: + needs: checks + runs-on: ${{ matrix.os }} + strategy: + max-parallel: 8 + matrix: + os: [ubuntu-latest, macos-latest, windows-latest, ubuntu-16.04] + python-version: [3.6, 3.7, 3.8, 3.9] + + steps: + - name: Set up Python ${{ matrix.python-version }} x64 + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + architecture: x64 + + - uses: actions/checkout@v2 + + - name: Install tox + run: | + pip install --upgrade pip + pip install tox + + - name: Test with tox + run: tox -e py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..57c68e5 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,38 @@ +name: release + +on: + push: + tags: + - v* + +jobs: + + upload: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.9 + + - name: Install dependencies + run: | + pip install --upgrade pip + pip install -r requirements.txt + + - name: Create source dist + run: python setup.py sdist + + - name: Create wheel dist + run: python setup.py bdist_wheel + + - name: Upload with twine + env: + TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }} + TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} + run: | + ls -l dist/* + twine upload dist/* diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 5e5c760..0000000 --- a/.travis.yml +++ /dev/null @@ -1,19 +0,0 @@ -sudo: false -language: python -install: python -m pip install tox -script: python -m tox -e py -matrix: - include: - - python: 3.5 - env: TOXENV=py35 - - python: 3.6 - env: TOXENV=py36 - - python: 3.7 - dist: xenial - env: TOXENV=py37 - - python: 3.8 - dist: xenial - env: TOXENV=py38 - - python: 3.8 - dist: xenial - env: TOXENV=pylint diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index 1f52a08..0000000 --- a/appveyor.yml +++ /dev/null @@ -1,22 +0,0 @@ -environment: - - matrix: - - - PYTHON: "C:\\Python35" - - PYTHON: "C:\\Python36" - - PYTHON: "C:\\Python37" - - PYTHON: "C:\\Python38" - - PYTHON: "C:\\Python35-x64" - - PYTHON: "C:\\Python36-x64" - - PYTHON: "C:\\Python37-x64" - - PYTHON: "C:\\Python38-x64" - -install: - - - "%PYTHON%\\python.exe -m pip install tox" - -build: off - -test_script: - - - "%PYTHON%\\python.exe -m tox -e py" diff --git a/tox.ini b/tox.ini index b54e574..6bbe69c 100644 --- a/tox.ini +++ b/tox.ini @@ -1,43 +1,93 @@ [tox] -envlist=py35,py36,py37,py38,pylint +envlist=bluecheck,doc8,docs,isortcheck,flake8,mypy,pylint,rstcheck,py36,py37,py38,py39 skip_missing_interpreters=True [testenv] +commands=pytest deps= django==2.2.* pytest pytest-django pytest-xdist -commands=python -m pytest setenv= DJANGO_SETTINGS_MODULE=tests.settings - PYTHONPATH={toxinidir} + +[testenv:blue] +commands=blue {toxinidir}/setup.py {toxinidir}/diskcache {toxinidir}/tests +deps=blue + +[testenv:bluecheck] +commands=blue --check {toxinidir}/setup.py {toxinidir}/diskcache {toxinidir}/tests +deps=blue + +[testenv:doc8] +deps=doc8 +commands=doc8 docs + +[testenv:docs] +allowlist_externals=make +changedir=docs +commands=make html +deps=sphinx + +[testenv:flake8] +commands=flake8 {toxinidir}/setup.py {toxinidir}/diskcache {toxinidir}/tests +deps=flake8 + +[testenv:isort] +commands=isort {toxinidir}/setup.py {toxinidir}/diskcache {toxinidir}/tests +deps=isort + +[testenv:isortcheck] +commands=isort --check {toxinidir}/setup.py {toxinidir}/diskcache {toxinidir}/tests +deps=isort + +[testenv:mypy] +commands=mypy {toxinidir}/diskcache +deps=mypy + +[testenv:pylint] +commands=pylint {toxinidir}/diskcache +deps=pylint + +[testenv:rstcheck] +commands=rstcheck {toxinidir}/README.rst +deps=rstcheck + +[testenv:uploaddocs] +allowlist_externals=rsync +changedir=docs +commands= + rsync -azP --stats --delete _build/html/ \ + grantjenks.com:/srv/www/www.grantjenks.com/public/docs/diskcache/ + +[isort] +multi_line_output = 3 +include_trailing_comma = True +force_grid_wrap = 0 +use_parentheses = True +ensure_newline_before_comments = True +line_length = 79 [pytest] addopts= -n auto + --cov-branch + --cov-fail-under=100 + --cov-report=term-missing + --cov=diskcache + --doctest-glob="*.rst" --ignore tests/benchmark_core.py --ignore tests/benchmark_djangocache.py --ignore tests/benchmark_glob.py --ignore tests/issue_85.py --ignore tests/plot.py -norecursedirs=site-packages -testpaths=docs diskcache tests - -[testenv:pylint] -deps= - django==2.2.* - flake8 - pylint -commands= - flake8 diskcache - pylint diskcache [doc8] -ignore=D000 +# ignore=D000 [flake8] -ignore= - E124 - E303 - W503 +# ignore= +# E124 +# E303 +# W503 From 8d2009abd1e92875a31b21bdf2799ea757fb6b24 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Fri, 22 Jan 2021 11:19:45 -0800 Subject: [PATCH 026/126] Rewrite k/v store benchmarking to avoid IPython "magic" syntax --- tests/benchmark_kv_store.py | 110 +++++++++++++++++++++++------------- 1 file changed, 70 insertions(+), 40 deletions(-) diff --git a/tests/benchmark_kv_store.py b/tests/benchmark_kv_store.py index d1459e7..e214f1a 100644 --- a/tests/benchmark_kv_store.py +++ b/tests/benchmark_kv_store.py @@ -1,48 +1,78 @@ -import dbm +"""Benchmarking Key-Value Stores + +$ python -m IPython tests/benchmark_kv_store.py + +""" + import diskcache -import pickledb -import shelve -import sqlitedict -import timeit + +from IPython import get_ipython + +ipython = get_ipython() +assert ipython is not None, 'No IPython! Run with $ ipython ...' value = 'value' print('diskcache set') dc = diskcache.FanoutCache('/tmp/diskcache') -%timeit -n 100 -r 7 dc['key'] = value +ipython.magic("timeit -n 100 -r 7 dc['key'] = value") print('diskcache get') -%timeit -n 100 -r 7 dc['key'] +ipython.magic("timeit -n 100 -r 7 dc['key']") print('diskcache set/delete') -%timeit -n 100 -r 7 dc['key'] = value; del dc['key'] - -print('dbm set') -d = dbm.open('/tmp/dbm', 'c') -%timeit -n 100 -r 7 d['key'] = value; d.sync() -print('dbm get') -%timeit -n 100 -r 7 d['key'] -print('dbm set/delete') -%timeit -n 100 -r 7 d['key'] = value; del d['key']; d.sync() - -print('shelve set') -s = shelve.open('/tmp/shelve') -%timeit -n 100 -r 7 s['key'] = value; s.sync() -print('shelve get') -%timeit -n 100 -r 7 s['key'] -print('shelve set/delete') -%timeit -n 100 -r 7 s['key'] = value; del s['key']; s.sync() - -print('sqlitedict set') -sd = sqlitedict.SqliteDict('/tmp/sqlitedict', autocommit=True) -%timeit -n 100 -r 7 sd['key'] = value -print('sqlitedict get') -%timeit -n 100 -r 7 sd['key'] -print('sqlitedict set/delete') -%timeit -n 100 -r 7 sd['key'] = value; del sd['key'] - -print('pickledb set') -p = pickledb.load('/tmp/pickledb', True) -%timeit -n 100 -r 7 p['key'] = value -print('pickledb get') -%timeit -n 100 -r 7 p = pickledb.load('/tmp/pickledb', True); p['key'] -print('pickledb set/delete') -%timeit -n 100 -r 7 p['key'] = value; del p['key'] +ipython.magic("timeit -n 100 -r 7 dc['key'] = value; del dc['key']") + +try: + import dbm.gnu # Only trust GNU DBM +except ImportError: + print('Error: Cannot import dbm.gnu') + print('Error: Skipping import shelve') +else: + print('dbm set') + d = dbm.gnu.open('/tmp/dbm', 'c') + ipython.magic("timeit -n 100 -r 7 d['key'] = value; d.sync()") + print('dbm get') + ipython.magic("timeit -n 100 -r 7 d['key']") + print('dbm set/delete') + ipython.magic( + "timeit -n 100 -r 7 d['key'] = value; d.sync(); del d['key']; d.sync()" + ) + + import shelve + + print('shelve set') + s = shelve.open('/tmp/shelve') + ipython.magic("timeit -n 100 -r 7 s['key'] = value; s.sync()") + print('shelve get') + ipython.magic("timeit -n 100 -r 7 s['key']") + print('shelve set/delete') + ipython.magic( + "timeit -n 100 -r 7 s['key'] = value; s.sync(); del s['key']; s.sync()" + ) + +try: + import sqlitedict +except ImportError: + print('Error: Cannot import sqlitedict') +else: + print('sqlitedict set') + sd = sqlitedict.SqliteDict('/tmp/sqlitedict', autocommit=True) + ipython.magic("timeit -n 100 -r 7 sd['key'] = value") + print('sqlitedict get') + ipython.magic("timeit -n 100 -r 7 sd['key']") + print('sqlitedict set/delete') + ipython.magic("timeit -n 100 -r 7 sd['key'] = value; del sd['key']") + +try: + import pickledb +except ImportError: + print('Error: Cannot import pickledb') +else: + print('pickledb set') + p = pickledb.load('/tmp/pickledb', True) + ipython.magic("timeit -n 100 -r 7 p['key'] = value") + print('pickledb get') + ipython.magic( + "timeit -n 100 -r 7 p = pickledb.load('/tmp/pickledb', True); p['key']" + ) + print('pickledb set/delete') + ipython.magic("timeit -n 100 -r 7 p['key'] = value; del p['key']") From b7ecbe9e9d9d51d78b49058c6bea68400ff1ecf5 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Fri, 22 Jan 2021 11:23:00 -0800 Subject: [PATCH 027/126] Update development requirements for editable install --- requirements.txt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 835f48a..a9023a1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,12 @@ +-e . +blue coverage django==2.2.* django_redis doc8 -gj +flake8 +ipython +pickleDB pylibmc pylint pytest @@ -12,6 +16,7 @@ pytest-env pytest-xdist rstcheck sphinx +sqlitedict tox twine wheel From 2ed7787817d1a7587dba077e972faea16fb6f5d8 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Fri, 22 Jan 2021 11:25:53 -0800 Subject: [PATCH 028/126] I blue it --- diskcache/__init__.py | 8 +- diskcache/core.py | 323 +++++++++++++++++---------------- diskcache/djangocache.py | 89 +++++---- diskcache/fanout.py | 59 ++---- diskcache/persistent.py | 62 +------ diskcache/recipes.py | 63 +++++-- setup.py | 2 + tests/benchmark_core.py | 90 ++++++--- tests/benchmark_djangocache.py | 23 ++- tests/benchmark_glob.py | 6 +- tests/issue_109.py | 3 +- tests/issue_85.py | 1 + tests/models.py | 4 +- tests/plot.py | 23 ++- tests/plot_early_recompute.py | 8 + tests/settings_benchmark.py | 6 +- tests/stress_test_core.py | 116 +++++++++--- tests/stress_test_fanout.py | 116 +++++++++--- tests/test_core.py | 12 +- tests/test_djangocache.py | 245 ++++++++++++++++--------- tests/test_recipes.py | 6 + tests/utils.py | 23 ++- 22 files changed, 787 insertions(+), 501 deletions(-) diff --git a/diskcache/__init__.py b/diskcache/__init__.py index e4d747b..0300123 100644 --- a/diskcache/__init__.py +++ b/diskcache/__init__.py @@ -7,7 +7,12 @@ """ from .core import ( - Cache, Disk, EmptyDirWarning, JSONDisk, UnknownFileWarning, Timeout + Cache, + Disk, + EmptyDirWarning, + JSONDisk, + UnknownFileWarning, + Timeout, ) from .core import DEFAULT_SETTINGS, ENOVAL, EVICTION_POLICY, UNKNOWN from .fanout import FanoutCache @@ -40,6 +45,7 @@ try: from .djangocache import DjangoCache # noqa + __all__.append('DjangoCache') except Exception: # pylint: disable=broad-except # Django not installed or not setup so ignore. diff --git a/diskcache/core.py b/diskcache/core.py index 1fae11b..419c64f 100644 --- a/diskcache/core.py +++ b/diskcache/core.py @@ -29,12 +29,14 @@ def full_name(func): try: WindowsError except NameError: + class WindowsError(Exception): "Windows error place-holder on platforms without support." class Constant(tuple): "Pretty display of immutable constant." + def __new__(cls, name): return tuple.__new__(cls, (name,)) @@ -54,15 +56,15 @@ def __repr__(self): DEFAULT_SETTINGS = { u'statistics': 0, # False - u'tag_index': 0, # False + u'tag_index': 0, # False u'eviction_policy': u'least-recently-stored', u'size_limit': 2 ** 30, # 1gb u'cull_limit': 10, - u'sqlite_auto_vacuum': 1, # FULL - u'sqlite_cache_size': 2 ** 13, # 8,192 pages + u'sqlite_auto_vacuum': 1, # FULL + u'sqlite_cache_size': 2 ** 13, # 8,192 pages u'sqlite_journal_mode': u'wal', - u'sqlite_mmap_size': 2 ** 26, # 64mb - u'sqlite_synchronous': 1, # NORMAL + u'sqlite_mmap_size': 2 ** 26, # 64mb + u'sqlite_synchronous': 1, # NORMAL u'disk_min_file_size': 2 ** 15, # 32kb u'disk_pickle_protocol': pickle.HIGHEST_PROTOCOL, } @@ -109,6 +111,7 @@ def __repr__(self): class Disk(object): "Cache key and value serialization for SQLite database and files." + def __init__(self, directory, min_file_size=0, pickle_protocol=0): """Initialize disk instance. @@ -121,7 +124,6 @@ def __init__(self, directory, min_file_size=0, pickle_protocol=0): self.min_file_size = min_file_size self.pickle_protocol = pickle_protocol - def hash(self, key): """Compute portable hash for `key`. @@ -143,7 +145,6 @@ def hash(self, key): assert type_disk_key is float return zlib.adler32(struct.pack('!d', disk_key)) & mask - def put(self, key): """Convert `key` to fields key and raw for Cache table. @@ -156,17 +157,20 @@ def put(self, key): if type_key is bytes: return sqlite3.Binary(key), True - elif ((type_key is str) - or (type_key is int - and -9223372036854775808 <= key <= 9223372036854775807) - or (type_key is float)): + elif ( + (type_key is str) + or ( + type_key is int + and -9223372036854775808 <= key <= 9223372036854775807 + ) + or (type_key is float) + ): return key, True else: data = pickle.dumps(key, protocol=self.pickle_protocol) result = pickletools.optimize(data) return sqlite3.Binary(result), False - def get(self, key, raw): """Convert fields `key` and `raw` from Cache table to key. @@ -181,7 +185,6 @@ def get(self, key, raw): else: return pickle.load(io.BytesIO(key)) - def store(self, value, read, key=UNKNOWN): """Convert `value` to fields size, mode, filename, and value for Cache table. @@ -196,10 +199,14 @@ def store(self, value, read, key=UNKNOWN): type_value = type(value) min_file_size = self.min_file_size - if ((type_value is str and len(value) < min_file_size) - or (type_value is int - and -9223372036854775808 <= value <= 9223372036854775807) - or (type_value is float)): + if ( + (type_value is str and len(value) < min_file_size) + or ( + type_value is int + and -9223372036854775808 <= value <= 9223372036854775807 + ) + or (type_value is float) + ): return 0, MODE_RAW, None, value elif type_value is bytes: if len(value) < min_file_size: @@ -243,7 +250,6 @@ def store(self, value, read, key=UNKNOWN): return len(result), MODE_PICKLE, filename, None - def fetch(self, mode, filename, value, read): """Convert fields `mode`, `filename`, and `value` from Cache table to value. @@ -275,7 +281,6 @@ def fetch(self, mode, filename, value, read): else: return pickle.load(io.BytesIO(value)) - def filename(self, key=UNKNOWN, value=UNKNOWN): """Return filename and full-path tuple for file storage. @@ -310,7 +315,6 @@ def filename(self, key=UNKNOWN, value=UNKNOWN): full_path = op.join(self._directory, filename) return filename, full_path - def remove(self, filename): """Remove a file given by `filename`. @@ -335,6 +339,7 @@ def remove(self, filename): class JSONDisk(Disk): "Cache key and value using JSON serialization with zlib compression." + def __init__(self, directory, compress_level=1, **kwargs): """Initialize JSON disk instance. @@ -351,25 +356,21 @@ def __init__(self, directory, compress_level=1, **kwargs): self.compress_level = compress_level super().__init__(directory, **kwargs) - def put(self, key): json_bytes = json.dumps(key).encode('utf-8') data = zlib.compress(json_bytes, self.compress_level) return super().put(data) - def get(self, key, raw): data = super().get(key, raw) return json.loads(zlib.decompress(data).decode('utf-8')) - def store(self, value, read, key=UNKNOWN): if not read: json_bytes = json.dumps(value).encode('utf-8') value = zlib.compress(json_bytes, self.compress_level) return super().store(value, read, key=key) - def fetch(self, mode, filename, value, read): data = super().fetch(mode, filename, value, read) if not read: @@ -419,6 +420,7 @@ def args_to_key(base, args, kwargs, typed): class Cache(object): "Disk and file backed cache." + def __init__(self, directory=None, timeout=60, disk=Disk, **settings): """Initialize cache instance. @@ -451,7 +453,7 @@ def __init__(self, directory=None, timeout=60, disk=Disk, **settings): raise EnvironmentError( error.errno, 'Cache directory "%s" does not exist' - ' and could not be created' % self._directory + ' and could not be created' % self._directory, ) from None sql = self._sql_retry @@ -459,9 +461,9 @@ def __init__(self, directory=None, timeout=60, disk=Disk, **settings): # Setup Settings table. try: - current_settings = dict(sql( - 'SELECT key, value FROM Settings' - ).fetchall()) + current_settings = dict( + sql('SELECT key, value FROM Settings').fetchall() + ) except sqlite3.OperationalError: current_settings = {} @@ -478,7 +480,8 @@ def __init__(self, directory=None, timeout=60, disk=Disk, **settings): if key.startswith('sqlite_'): self.reset(key, value, update=False) - sql('CREATE TABLE IF NOT EXISTS Settings (' + sql( + 'CREATE TABLE IF NOT EXISTS Settings (' ' key TEXT NOT NULL UNIQUE,' ' value)' ) @@ -486,7 +489,8 @@ def __init__(self, directory=None, timeout=60, disk=Disk, **settings): # Setup Disk object (must happen after settings initialized). kwargs = { - key[5:]: value for key, value in sets.items() + key[5:]: value + for key, value in sets.items() if key.startswith('disk_') } self._disk = disk(directory, **kwargs) @@ -503,11 +507,12 @@ def __init__(self, directory=None, timeout=60, disk=Disk, **settings): sql(query, (key, value)) self.reset(key) - (self._page_size,), = sql('PRAGMA page_size').fetchall() + ((self._page_size,),) = sql('PRAGMA page_size').fetchall() # Setup Cache table. - sql('CREATE TABLE IF NOT EXISTS Cache (' + sql( + 'CREATE TABLE IF NOT EXISTS Cache (' ' rowid INTEGER PRIMARY KEY,' ' key BLOB,' ' raw INTEGER,' @@ -522,11 +527,13 @@ def __init__(self, directory=None, timeout=60, disk=Disk, **settings): ' value BLOB)' ) - sql('CREATE UNIQUE INDEX IF NOT EXISTS Cache_key_raw ON' + sql( + 'CREATE UNIQUE INDEX IF NOT EXISTS Cache_key_raw ON' ' Cache(key, raw)' ) - sql('CREATE INDEX IF NOT EXISTS Cache_expire_time ON' + sql( + 'CREATE INDEX IF NOT EXISTS Cache_expire_time ON' ' Cache (expire_time)' ) @@ -537,32 +544,37 @@ def __init__(self, directory=None, timeout=60, disk=Disk, **settings): # Use triggers to keep Metadata updated. - sql('CREATE TRIGGER IF NOT EXISTS Settings_count_insert' + sql( + 'CREATE TRIGGER IF NOT EXISTS Settings_count_insert' ' AFTER INSERT ON Cache FOR EACH ROW BEGIN' ' UPDATE Settings SET value = value + 1' ' WHERE key = "count"; END' ) - sql('CREATE TRIGGER IF NOT EXISTS Settings_count_delete' + sql( + 'CREATE TRIGGER IF NOT EXISTS Settings_count_delete' ' AFTER DELETE ON Cache FOR EACH ROW BEGIN' ' UPDATE Settings SET value = value - 1' ' WHERE key = "count"; END' ) - sql('CREATE TRIGGER IF NOT EXISTS Settings_size_insert' + sql( + 'CREATE TRIGGER IF NOT EXISTS Settings_size_insert' ' AFTER INSERT ON Cache FOR EACH ROW BEGIN' ' UPDATE Settings SET value = value + NEW.size' ' WHERE key = "size"; END' ) - sql('CREATE TRIGGER IF NOT EXISTS Settings_size_update' + sql( + 'CREATE TRIGGER IF NOT EXISTS Settings_size_update' ' AFTER UPDATE ON Cache FOR EACH ROW BEGIN' ' UPDATE Settings' ' SET value = value + NEW.size - OLD.size' ' WHERE key = "size"; END' ) - sql('CREATE TRIGGER IF NOT EXISTS Settings_size_delete' + sql( + 'CREATE TRIGGER IF NOT EXISTS Settings_size_delete' ' AFTER DELETE ON Cache FOR EACH ROW BEGIN' ' UPDATE Settings SET value = value - OLD.size' ' WHERE key = "size"; END' @@ -581,25 +593,21 @@ def __init__(self, directory=None, timeout=60, disk=Disk, **settings): self._timeout = timeout self._sql # pylint: disable=pointless-statement - @property def directory(self): """Cache directory.""" return self._directory - @property def timeout(self): """SQLite connection timeout value in seconds.""" return self._timeout - @property def disk(self): """Disk used for serialization.""" return self._disk - @property def _con(self): # Check process ID to support process forking. If the process @@ -638,12 +646,10 @@ def _con(self): return con - @property def _sql(self): return self._con.execute - @property def _sql_retry(self): sql = self._sql @@ -671,7 +677,6 @@ def _execute_with_retry(statement, *args, **kwargs): return _execute_with_retry - @cl.contextmanager def transact(self, retry=False): """Context manager to perform a transaction by locking the cache. @@ -703,7 +708,6 @@ def transact(self, retry=False): with self._transact(retry=retry): yield - @cl.contextmanager def _transact(self, retry=False, filename=None): sql = self._sql @@ -745,7 +749,6 @@ def _transact(self, retry=False, filename=None): if name is not None: _disk_remove(name) - def set(self, key, value, expire=None, read=False, tag=None, retry=False): """Set `key` and `value` item in cache. @@ -801,7 +804,7 @@ def set(self, key, value, expire=None, read=False, tag=None, retry=False): ).fetchall() if rows: - (rowid, old_filename), = rows + ((rowid, old_filename),) = rows cleanup(old_filename) self._row_update(rowid, now, columns) else: @@ -811,7 +814,6 @@ def set(self, key, value, expire=None, read=False, tag=None, retry=False): return True - def __setitem__(self, key, value): """Set corresponding `value` for `key` in cache. @@ -823,11 +825,11 @@ def __setitem__(self, key, value): """ self.set(key, value, retry=True) - def _row_update(self, rowid, now, columns): sql = self._sql expire_time, tag, size, mode, filename, value = columns - sql('UPDATE Cache SET' + sql( + 'UPDATE Cache SET' ' store_time = ?,' ' expire_time = ?,' ' access_time = ?,' @@ -837,11 +839,12 @@ def _row_update(self, rowid, now, columns): ' mode = ?,' ' filename = ?,' ' value = ?' - ' WHERE rowid = ?', ( - now, # store_time + ' WHERE rowid = ?', + ( + now, # store_time expire_time, - now, # access_time - 0, # access_count + now, # access_time + 0, # access_count tag, size, mode, @@ -851,20 +854,21 @@ def _row_update(self, rowid, now, columns): ), ) - def _row_insert(self, key, raw, now, columns): sql = self._sql expire_time, tag, size, mode, filename, value = columns - sql('INSERT INTO Cache(' + sql( + 'INSERT INTO Cache(' ' key, raw, store_time, expire_time, access_time,' ' access_count, tag, size, mode, filename, value' - ') VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', ( + ') VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', + ( key, raw, - now, # store_time + now, # store_time expire_time, - now, # access_time - 0, # access_count + now, # access_time + 0, # access_count tag, size, mode, @@ -873,7 +877,6 @@ def _row_insert(self, key, raw, now, columns): ), ) - def _cull(self, now, sql, cleanup, limit=None): cull_limit = self.cull_limit if limit is None else limit @@ -892,13 +895,12 @@ def _cull(self, now, sql, cleanup, limit=None): rows = sql(select_expired, (now, cull_limit)).fetchall() if rows: - delete_expired = ( - 'DELETE FROM Cache WHERE rowid IN (%s)' - % (select_expired_template % 'rowid') + delete_expired = 'DELETE FROM Cache WHERE rowid IN (%s)' % ( + select_expired_template % 'rowid' ) sql(delete_expired, (now, cull_limit)) - for filename, in rows: + for (filename,) in rows: cleanup(filename) cull_limit -= len(rows) @@ -917,16 +919,14 @@ def _cull(self, now, sql, cleanup, limit=None): rows = sql(select_filename, (cull_limit,)).fetchall() if rows: - delete = ( - 'DELETE FROM Cache WHERE rowid IN (%s)' - % (select_policy.format(fields='rowid', now=now)) + delete = 'DELETE FROM Cache WHERE rowid IN (%s)' % ( + select_policy.format(fields='rowid', now=now) ) sql(delete, (cull_limit,)) - for filename, in rows: + for (filename,) in rows: cleanup(filename) - def touch(self, key, expire=None, retry=False): """Touch `key` in cache and update `expire` time. @@ -953,17 +953,17 @@ def touch(self, key, expire=None, retry=False): ).fetchall() if rows: - (rowid, old_expire_time), = rows + ((rowid, old_expire_time),) = rows if old_expire_time is None or old_expire_time > now: - sql('UPDATE Cache SET expire_time = ? WHERE rowid = ?', + sql( + 'UPDATE Cache SET expire_time = ? WHERE rowid = ?', (expire_time, rowid), ) return True return False - def add(self, key, value, expire=None, read=False, tag=None, retry=False): """Add `key` and `value` item to cache. @@ -1003,7 +1003,7 @@ def add(self, key, value, expire=None, read=False, tag=None, retry=False): ).fetchall() if rows: - (rowid, old_filename, old_expire_time), = rows + ((rowid, old_filename, old_expire_time),) = rows if old_expire_time is None or old_expire_time > now: cleanup(filename) @@ -1018,7 +1018,6 @@ def add(self, key, value, expire=None, read=False, tag=None, retry=False): return True - def incr(self, key, delta=1, default=0, retry=False): """Increment value by delta for item with key. @@ -1059,22 +1058,22 @@ def incr(self, key, delta=1, default=0, retry=False): raise KeyError(key) value = default + delta - columns = ( - (None, None) + self._disk.store(value, False, key=key) + columns = (None, None) + self._disk.store( + value, False, key=key ) self._row_insert(db_key, raw, now, columns) self._cull(now, sql, cleanup) return value - (rowid, expire_time, filename, value), = rows + ((rowid, expire_time, filename, value),) = rows if expire_time is not None and expire_time < now: if default is None: raise KeyError(key) value = default + delta - columns = ( - (None, None) + self._disk.store(value, False, key=key) + columns = (None, None) + self._disk.store( + value, False, key=key ) self._row_update(rowid, now, columns) self._cull(now, sql, cleanup) @@ -1094,7 +1093,6 @@ def incr(self, key, delta=1, default=0, retry=False): return value - def decr(self, key, delta=1, default=0, retry=False): """Decrement value by delta for item with key. @@ -1125,9 +1123,15 @@ def decr(self, key, delta=1, default=0, retry=False): """ return self.incr(key, -delta, default, retry) - - def get(self, key, default=None, read=False, expire_time=False, tag=False, - retry=False): + def get( + self, + key, + default=None, + read=False, + expire_time=False, + tag=False, + retry=False, + ): """Retrieve value from cache. If `key` is missing, return `default`. Raises :exc:`Timeout` error when database timeout occurs and `retry` is @@ -1166,7 +1170,7 @@ def get(self, key, default=None, read=False, expire_time=False, tag=False, if not rows: return default - (rowid, db_expire_time, db_tag, mode, filename, db_value), = rows + ((rowid, db_expire_time, db_tag, mode, filename, db_value),) = rows try: value = self._disk.fetch(mode, filename, db_value, read) @@ -1190,8 +1194,9 @@ def get(self, key, default=None, read=False, expire_time=False, tag=False, sql(cache_miss) return default - (rowid, db_expire_time, db_tag, - mode, filename, db_value), = rows # noqa: E127 + ( + (rowid, db_expire_time, db_tag, mode, filename, db_value), + ) = rows # noqa: E127 try: value = self._disk.fetch(mode, filename, db_value, read) @@ -1222,7 +1227,6 @@ def get(self, key, default=None, read=False, expire_time=False, tag=False, else: return value - def __getitem__(self, key): """Return corresponding value for `key` from cache. @@ -1236,7 +1240,6 @@ def __getitem__(self, key): raise KeyError(key) return value - def read(self, key, retry=False): """Return file handle value corresponding to `key` from cache. @@ -1255,7 +1258,6 @@ def read(self, key, retry=False): raise KeyError(key) return handle - def __contains__(self, key): """Return `True` if `key` matching item is found in cache. @@ -1275,8 +1277,9 @@ def __contains__(self, key): return bool(rows) - - def pop(self, key, default=None, expire_time=False, tag=False, retry=False): # noqa: E501 + def pop( + self, key, default=None, expire_time=False, tag=False, retry=False + ): # noqa: E501 """Remove corresponding item for `key` from cache and return value. If `key` is missing, return `default`. @@ -1314,7 +1317,7 @@ def pop(self, key, default=None, expire_time=False, tag=False, retry=False): # if not rows: return default - (rowid, db_expire_time, db_tag, mode, filename, db_value), = rows + ((rowid, db_expire_time, db_tag, mode, filename, db_value),) = rows sql('DELETE FROM Cache WHERE rowid = ?', (rowid,)) @@ -1339,7 +1342,6 @@ def pop(self, key, default=None, expire_time=False, tag=False, retry=False): # else: return value - def __delitem__(self, key, retry=True): """Delete corresponding item for `key` from cache. @@ -1365,13 +1367,12 @@ def __delitem__(self, key, retry=True): if not rows: raise KeyError(key) - (rowid, filename), = rows + ((rowid, filename),) = rows sql('DELETE FROM Cache WHERE rowid = ?', (rowid,)) cleanup(filename) return True - def delete(self, key, retry=False): """Delete corresponding item for `key` from cache. @@ -1391,9 +1392,16 @@ def delete(self, key, retry=False): except KeyError: return False - - def push(self, value, prefix=None, side='back', expire=None, read=False, - tag=None, retry=False): + def push( + self, + value, + prefix=None, + side='back', + expire=None, + read=False, + tag=None, + retry=False, + ): """Push `value` onto `side` of queue identified by `prefix` in cache. When prefix is None, integer keys are used. Otherwise, string keys are @@ -1459,10 +1467,10 @@ def push(self, value, prefix=None, side='back', expire=None, read=False, rows = sql(select, (min_key, max_key, raw)).fetchall() if rows: - (key,), = rows + ((key,),) = rows if prefix is not None: - num = int(key[(key.rfind('-') + 1):]) + num = int(key[(key.rfind('-') + 1) :]) else: num = key @@ -1484,9 +1492,15 @@ def push(self, value, prefix=None, side='back', expire=None, read=False, return db_key - - def pull(self, prefix=None, default=(None, None), side='front', - expire_time=False, tag=False, retry=False): + def pull( + self, + prefix=None, + default=(None, None), + side='front', + expire_time=False, + tag=False, + retry=False, + ): """Pull key and value item pair from `side` of queue in cache. When prefix is None, integer keys are used. Otherwise, string keys are @@ -1567,8 +1581,9 @@ def pull(self, prefix=None, default=(None, None), side='front', if not rows: return default - (rowid, key, db_expire, db_tag, mode, name, - db_value), = rows + ( + (rowid, key, db_expire, db_tag, mode, name, db_value), + ) = rows sql('DELETE FROM Cache WHERE rowid = ?', (rowid,)) @@ -1598,9 +1613,15 @@ def pull(self, prefix=None, default=(None, None), side='front', else: return key, value - - def peek(self, prefix=None, default=(None, None), side='front', - expire_time=False, tag=False, retry=False): + def peek( + self, + prefix=None, + default=(None, None), + side='front', + expire_time=False, + tag=False, + retry=False, + ): """Peek at key and value item pair from `side` of queue in cache. When prefix is None, integer keys are used. Otherwise, string keys are @@ -1677,8 +1698,9 @@ def peek(self, prefix=None, default=(None, None), side='front', if not rows: return default - (rowid, key, db_expire, db_tag, mode, name, - db_value), = rows + ( + (rowid, key, db_expire, db_tag, mode, name, db_value), + ) = rows if db_expire is not None and db_expire < time.time(): sql('DELETE FROM Cache WHERE rowid = ?', (rowid,)) @@ -1707,7 +1729,6 @@ def peek(self, prefix=None, default=(None, None), side='front', else: return key, value - def peekitem(self, last=True, expire_time=False, tag=False, retry=False): """Peek at key and value item pair in cache based on iteration order. @@ -1749,8 +1770,18 @@ def peekitem(self, last=True, expire_time=False, tag=False, retry=False): if not rows: raise KeyError('dictionary is empty') - (rowid, db_key, raw, db_expire, db_tag, mode, name, - db_value), = rows + ( + ( + rowid, + db_key, + raw, + db_expire, + db_tag, + mode, + name, + db_value, + ), + ) = rows if db_expire is not None and db_expire < time.time(): sql('DELETE FROM Cache WHERE rowid = ?', (rowid,)) @@ -1778,7 +1809,6 @@ def peekitem(self, last=True, expire_time=False, tag=False, retry=False): else: return key, value - def memoize(self, name=None, typed=False, expire=None, tag=None): """Memoizing cache decorator. @@ -1871,7 +1901,6 @@ def __cache_key__(*args, **kwargs): return decorator - def check(self, fix=False, retry=False): """Check database and file system consistency. @@ -1901,7 +1930,7 @@ def check(self, fix=False, retry=False): rows = sql('PRAGMA integrity_check').fetchall() if len(rows) != 1 or rows[0][0] != u'ok': - for message, in rows: + for (message,) in rows: warnings.warn(message) if fix: @@ -1932,7 +1961,8 @@ def check(self, fix=False, retry=False): warnings.warn(message % args) if fix: - sql('UPDATE Cache SET size = ?' + sql( + 'UPDATE Cache SET size = ?' ' WHERE rowid = ?', (real_size, rowid), ) @@ -1973,14 +2003,15 @@ def check(self, fix=False, retry=False): # Check Settings.count against count of Cache rows. self.reset('count') - (count,), = sql('SELECT COUNT(key) FROM Cache').fetchall() + ((count,),) = sql('SELECT COUNT(key) FROM Cache').fetchall() if self.count != count: message = 'Settings.count != COUNT(Cache.key); %d != %d' warnings.warn(message % (self.count, count)) if fix: - sql('UPDATE Settings SET value = ? WHERE key = ?', + sql( + 'UPDATE Settings SET value = ? WHERE key = ?', (count, 'count'), ) @@ -1988,20 +2019,20 @@ def check(self, fix=False, retry=False): self.reset('size') select_size = 'SELECT COALESCE(SUM(size), 0) FROM Cache' - (size,), = sql(select_size).fetchall() + ((size,),) = sql(select_size).fetchall() if self.size != size: message = 'Settings.size != SUM(Cache.size); %d != %d' warnings.warn(message % (self.size, size)) if fix: - sql('UPDATE Settings SET value = ? WHERE key =?', + sql( + 'UPDATE Settings SET value = ? WHERE key =?', (size, 'size'), ) return warns - def create_tag_index(self): """Create tag index on cache database. @@ -2014,7 +2045,6 @@ def create_tag_index(self): sql('CREATE INDEX IF NOT EXISTS Cache_tag_rowid ON Cache(tag, rowid)') self.reset('tag_index', 1) - def drop_tag_index(self): """Drop tag index on cache database. @@ -2025,7 +2055,6 @@ def drop_tag_index(self): sql('DROP INDEX IF EXISTS Cache_tag_rowid') self.reset('tag_index', 0) - def evict(self, tag, retry=False): """Remove items with matching `tag` from cache. @@ -2053,7 +2082,6 @@ def evict(self, tag, retry=False): args = [tag, 0, 100] return self._select_delete(select, args, arg_index=1, retry=retry) - def expire(self, now=None, retry=False): """Remove expired items from cache. @@ -2081,7 +2109,6 @@ def expire(self, now=None, retry=False): args = [0, now or time.time(), 100] return self._select_delete(select, args, row_index=1, retry=retry) - def cull(self, retry=False): """Cull items from cache until volume is less than size limit. @@ -2130,14 +2157,13 @@ def cull(self, retry=False): ) sql(delete, (10,)) - for filename, in rows: + for (filename,) in rows: cleanup(filename) except Timeout: raise Timeout(count) from None return count - def clear(self, retry=False): """Remove all items from cache. @@ -2164,9 +2190,9 @@ def clear(self, retry=False): args = [0, 100] return self._select_delete(select, args, retry=retry) - - def _select_delete(self, select, args, row_index=0, arg_index=0, - retry=False): + def _select_delete( + self, select, args, row_index=0, arg_index=0, retry=False + ): count = 0 delete = 'DELETE FROM Cache WHERE rowid IN (%s)' @@ -2190,7 +2216,6 @@ def _select_delete(self, select, args, row_index=0, arg_index=0, return count - def iterkeys(self, reverse=False): """Iterate Cache keys in database sort order. @@ -2234,7 +2259,7 @@ def iterkeys(self, reverse=False): row = sql(select).fetchall() if row: - (key, raw), = row + ((key, raw),) = row else: return @@ -2249,11 +2274,10 @@ def iterkeys(self, reverse=False): for key, raw in rows: yield _disk_get(key, raw) - def _iter(self, ascending=True): sql = self._sql rows = sql('SELECT MAX(rowid) FROM Cache').fetchall() - (max_rowid,), = rows + ((max_rowid,),) = rows yield # Signal ready. if max_rowid is None: @@ -2283,21 +2307,18 @@ def _iter(self, ascending=True): for rowid, key, raw in rows: yield _disk_get(key, raw) - def __iter__(self): "Iterate keys in cache including expired items." iterator = self._iter() next(iterator) return iterator - def __reversed__(self): "Reverse iterate keys in cache including expired items." iterator = self._iter(ascending=False) next(iterator) return iterator - def stats(self, enable=True, reset=False): """Return cache statistics hits and misses. @@ -2317,22 +2338,18 @@ def stats(self, enable=True, reset=False): return result - def volume(self): """Return estimated total size of cache on disk. :return: size in bytes """ - (page_count,), = self._sql('PRAGMA page_count').fetchall() + ((page_count,),) = self._sql('PRAGMA page_count').fetchall() total_size = self._page_size * page_count + self.reset('size') return total_size - def close(self): - """Close database connection. - - """ + """Close database connection.""" con = getattr(self._local, 'con', None) if con is None: @@ -2345,31 +2362,25 @@ def close(self): except AttributeError: pass - def __enter__(self): # Create connection in thread. # pylint: disable=unused-variable connection = self._con # noqa return self - def __exit__(self, *exception): self.close() - def __len__(self): "Count of items in cache including expired items." return self.reset('count') - def __getstate__(self): return (self.directory, self.timeout, type(self.disk)) - def __setstate__(self, state): self.__init__(*state) - def reset(self, key, value=ENOVAL, update=True): """Reset `key` and `value` item from Settings table. @@ -2403,7 +2414,7 @@ def reset(self, key, value=ENOVAL, update=True): if value is ENOVAL: select = 'SELECT value FROM Settings WHERE key = ?' - (value,), = sql_retry(select, (key,)).fetchall() + ((value,),) = sql_retry(select, (key,)).fetchall() setattr(self, key, value) return value @@ -2431,7 +2442,9 @@ def reset(self, key, value=ENOVAL, update=True): while True: try: try: - (old_value,), = sql('PRAGMA %s' % (pragma)).fetchall() + ((old_value,),) = sql( + 'PRAGMA %s' % (pragma) + ).fetchall() update = old_value != value except ValueError: update = True diff --git a/diskcache/djangocache.py b/diskcache/djangocache.py index 329b966..2f1db07 100644 --- a/diskcache/djangocache.py +++ b/diskcache/djangocache.py @@ -15,6 +15,7 @@ class DjangoCache(BaseCache): "Django-compatible disk and file backed cache." + def __init__(self, directory, params): """Initialize DjangoCache instance. @@ -28,13 +29,11 @@ def __init__(self, directory, params): options = params.get('OPTIONS', {}) self._cache = FanoutCache(directory, shards, timeout, **options) - @property def directory(self): """Cache directory.""" return self._cache.directory - def cache(self, name): """Return Cache with given `name` in subdirectory. @@ -44,7 +43,6 @@ def cache(self, name): """ return self._cache.cache(name) - def deque(self, name): """Return Deque with given `name` in subdirectory. @@ -54,7 +52,6 @@ def deque(self, name): """ return self._cache.deque(name) - def index(self, name): """Return Index with given `name` in subdirectory. @@ -64,9 +61,16 @@ def index(self, name): """ return self._cache.index(name) - - def add(self, key, value, timeout=DEFAULT_TIMEOUT, version=None, - read=False, tag=None, retry=True): + def add( + self, + key, + value, + timeout=DEFAULT_TIMEOUT, + version=None, + read=False, + tag=None, + retry=True, + ): """Set a value in the cache if the key does not already exist. If timeout is given, that timeout will be used for the key; otherwise the default cache timeout will be used. @@ -89,9 +93,16 @@ def add(self, key, value, timeout=DEFAULT_TIMEOUT, version=None, timeout = self.get_backend_timeout(timeout=timeout) return self._cache.add(key, value, timeout, read, tag, retry) - - def get(self, key, default=None, version=None, read=False, - expire_time=False, tag=False, retry=False): + def get( + self, + key, + default=None, + version=None, + read=False, + expire_time=False, + tag=False, + retry=False, + ): """Fetch a given key from the cache. If the key does not exist, return default, which itself defaults to None. @@ -111,7 +122,6 @@ def get(self, key, default=None, version=None, read=False, key = self.make_key(key, version=version) return self._cache.get(key, default, read, expire_time, tag, retry) - def read(self, key, version=None): """Return file handle corresponding to `key` from Cache. @@ -124,9 +134,16 @@ def read(self, key, version=None): key = self.make_key(key, version=version) return self._cache.read(key) - - def set(self, key, value, timeout=DEFAULT_TIMEOUT, version=None, - read=False, tag=None, retry=True): + def set( + self, + key, + value, + timeout=DEFAULT_TIMEOUT, + version=None, + read=False, + tag=None, + retry=True, + ): """Set a value in the cache. If timeout is given, that timeout will be used for the key; otherwise the default cache timeout will be used. @@ -146,7 +163,6 @@ def set(self, key, value, timeout=DEFAULT_TIMEOUT, version=None, timeout = self.get_backend_timeout(timeout=timeout) return self._cache.set(key, value, timeout, read, tag, retry) - def touch(self, key, timeout=DEFAULT_TIMEOUT, version=None, retry=True): """Touch a key in the cache. If timeout is given, that timeout will be used for the key; otherwise the default cache timeout will be used. @@ -164,9 +180,15 @@ def touch(self, key, timeout=DEFAULT_TIMEOUT, version=None, retry=True): timeout = self.get_backend_timeout(timeout=timeout) return self._cache.touch(key, timeout, retry) - - def pop(self, key, default=None, version=None, expire_time=False, - tag=False, retry=True): + def pop( + self, + key, + default=None, + version=None, + expire_time=False, + tag=False, + retry=True, + ): """Remove corresponding item for `key` from cache and return value. If `key` is missing, return `default`. @@ -186,7 +208,6 @@ def pop(self, key, default=None, version=None, expire_time=False, key = self.make_key(key, version=version) return self._cache.pop(key, default, expire_time, tag, retry) - def delete(self, key, version=None, retry=True): """Delete a key from the cache, failing silently. @@ -200,7 +221,6 @@ def delete(self, key, version=None, retry=True): key = self.make_key(key, version=version) self._cache.delete(key, retry) - def incr(self, key, delta=1, version=None, default=None, retry=True): """Increment value by delta for item with key. @@ -230,7 +250,6 @@ def incr(self, key, delta=1, version=None, default=None, retry=True): except KeyError: raise ValueError("Key '%s' not found" % key) from None - def decr(self, key, delta=1, version=None, default=None, retry=True): """Decrement value by delta for item with key. @@ -259,7 +278,6 @@ def decr(self, key, delta=1, version=None, default=None, retry=True): # pylint: disable=arguments-differ return self.incr(key, -delta, version, default, retry) - def has_key(self, key, version=None): """Returns True if the key is in the cache and has not expired. @@ -271,7 +289,6 @@ def has_key(self, key, version=None): key = self.make_key(key, version=version) return key in self._cache - def expire(self): """Remove expired items from cache. @@ -280,7 +297,6 @@ def expire(self): """ return self._cache.expire() - def stats(self, enable=True, reset=False): """Return cache statistics hits and misses. @@ -291,7 +307,6 @@ def stats(self, enable=True, reset=False): """ return self._cache.stats(enable=enable, reset=reset) - def create_tag_index(self): """Create tag index on cache database. @@ -302,7 +317,6 @@ def create_tag_index(self): """ self._cache.create_tag_index() - def drop_tag_index(self): """Drop tag index on cache database. @@ -311,7 +325,6 @@ def drop_tag_index(self): """ self._cache.drop_tag_index() - def evict(self, tag): """Remove items with matching `tag` from cache. @@ -321,7 +334,6 @@ def evict(self, tag): """ return self._cache.evict(tag) - def cull(self): """Cull items from cache until volume is less than size limit. @@ -330,18 +342,15 @@ def cull(self): """ return self._cache.cull() - def clear(self): "Remove *all* values from the cache at once." return self._cache.clear() - def close(self, **kwargs): "Close the cache connection." # pylint: disable=unused-argument self._cache.close() - def get_backend_timeout(self, timeout=DEFAULT_TIMEOUT): """Return seconds to expiration. @@ -356,9 +365,14 @@ def get_backend_timeout(self, timeout=DEFAULT_TIMEOUT): timeout = -1 return None if timeout is None else timeout - - def memoize(self, name=None, timeout=DEFAULT_TIMEOUT, version=None, - typed=False, tag=None): + def memoize( + self, + name=None, + timeout=DEFAULT_TIMEOUT, + version=None, + typed=False, + tag=None, + ): """Memoizing cache decorator. Decorator to wrap callable with memoizing function using cache. @@ -418,7 +432,12 @@ def wrapper(*args, **kwargs): ) if valid_timeout: self.set( - key, result, timeout, version, tag=tag, retry=True, + key, + result, + timeout, + version, + tag=tag, + retry=True, ) return result diff --git a/diskcache/fanout.py b/diskcache/fanout.py index 9870c06..cc40c07 100644 --- a/diskcache/fanout.py +++ b/diskcache/fanout.py @@ -15,8 +15,10 @@ class FanoutCache(object): "Cache that shards keys and values." - def __init__(self, directory=None, shards=8, timeout=0.010, disk=Disk, - **settings): + + def __init__( + self, directory=None, shards=8, timeout=0.010, disk=Disk, **settings + ): """Initialize cache instance. :param str directory: cache directory @@ -52,20 +54,17 @@ def __init__(self, directory=None, shards=8, timeout=0.010, disk=Disk, self._deques = {} self._indexes = {} - @property def directory(self): """Cache directory.""" return self._directory - def __getattr__(self, name): safe_names = {'timeout', 'disk'} valid_name = name in DEFAULT_SETTINGS or name in safe_names assert valid_name, 'cannot access {} in cache shard'.format(name) return getattr(self._shards[0], name) - @cl.contextmanager def transact(self, retry=True): """Context manager to perform a transaction by locking the cache. @@ -99,7 +98,6 @@ def transact(self, retry=True): stack.enter_context(shard_transaction) yield - def set(self, key, value, expire=None, read=False, tag=None, retry=False): """Set `key` and `value` item in cache. @@ -126,7 +124,6 @@ def set(self, key, value, expire=None, read=False, tag=None, retry=False): except Timeout: return False - def __setitem__(self, key, value): """Set `key` and `value` item in cache. @@ -140,7 +137,6 @@ def __setitem__(self, key, value): shard = self._shards[index] shard[key] = value - def touch(self, key, expire=None, retry=False): """Touch `key` in cache and update `expire` time. @@ -161,7 +157,6 @@ def touch(self, key, expire=None, retry=False): except Timeout: return False - def add(self, key, value, expire=None, read=False, tag=None, retry=False): """Add `key` and `value` item to cache. @@ -193,7 +188,6 @@ def add(self, key, value, expire=None, read=False, tag=None, retry=False): except Timeout: return False - def incr(self, key, delta=1, default=0, retry=False): """Increment value by delta for item with key. @@ -225,7 +219,6 @@ def incr(self, key, delta=1, default=0, retry=False): except Timeout: return None - def decr(self, key, delta=1, default=0, retry=False): """Decrement value by delta for item with key. @@ -260,9 +253,15 @@ def decr(self, key, delta=1, default=0, retry=False): except Timeout: return None - - def get(self, key, default=None, read=False, expire_time=False, tag=False, - retry=False): + def get( + self, + key, + default=None, + read=False, + expire_time=False, + tag=False, + retry=False, + ): """Retrieve value from cache. If `key` is missing, return `default`. If database timeout occurs then returns `default` unless `retry` is set @@ -286,7 +285,6 @@ def get(self, key, default=None, read=False, expire_time=False, tag=False, except (Timeout, sqlite3.OperationalError): return default - def __getitem__(self, key): """Return corresponding value for `key` from cache. @@ -301,7 +299,6 @@ def __getitem__(self, key): shard = self._shards[index] return shard[key] - def read(self, key): """Return file handle corresponding to `key` from cache. @@ -315,7 +312,6 @@ def read(self, key): raise KeyError(key) return handle - def __contains__(self, key): """Return `True` if `key` matching item is found in cache. @@ -327,8 +323,9 @@ def __contains__(self, key): shard = self._shards[index] return key in shard - - def pop(self, key, default=None, expire_time=False, tag=False, retry=False): # noqa: E501 + def pop( + self, key, default=None, expire_time=False, tag=False, retry=False + ): # noqa: E501 """Remove corresponding item for `key` from cache and return value. If `key` is missing, return `default`. @@ -354,7 +351,6 @@ def pop(self, key, default=None, expire_time=False, tag=False, retry=False): # except Timeout: return default - def delete(self, key, retry=False): """Delete corresponding item for `key` from cache. @@ -375,7 +371,6 @@ def delete(self, key, retry=False): except Timeout: return False - def __delitem__(self, key): """Delete corresponding item for `key` from cache. @@ -389,7 +384,6 @@ def __delitem__(self, key): shard = self._shards[index] del shard[key] - def check(self, fix=False, retry=False): """Check database and file system consistency. @@ -413,7 +407,6 @@ def check(self, fix=False, retry=False): warnings = (shard.check(fix, retry) for shard in self._shards) return functools.reduce(operator.iadd, warnings, []) - def expire(self, retry=False): """Remove expired items from cache. @@ -426,7 +419,6 @@ def expire(self, retry=False): """ return self._remove('expire', args=(time.time(),), retry=retry) - def create_tag_index(self): """Create tag index on cache database. @@ -438,7 +430,6 @@ def create_tag_index(self): for shard in self._shards: shard.create_tag_index() - def drop_tag_index(self): """Drop tag index on cache database. @@ -448,7 +439,6 @@ def drop_tag_index(self): for shard in self._shards: shard.drop_tag_index() - def evict(self, tag, retry=False): """Remove items with matching `tag` from cache. @@ -462,7 +452,6 @@ def evict(self, tag, retry=False): """ return self._remove('evict', args=(tag,), retry=retry) - def cull(self, retry=False): """Cull items from cache until volume is less than size limit. @@ -475,7 +464,6 @@ def cull(self, retry=False): """ return self._remove('cull', retry=retry) - def clear(self, retry=False): """Remove all items from cache. @@ -488,7 +476,6 @@ def clear(self, retry=False): """ return self._remove('clear', retry=retry) - def _remove(self, name, args=(), retry=False): total = 0 for shard in self._shards: @@ -503,7 +490,6 @@ def _remove(self, name, args=(), retry=False): break return total - def stats(self, enable=True, reset=False): """Return cache statistics hits and misses. @@ -517,7 +503,6 @@ def stats(self, enable=True, reset=False): total_misses = sum(misses for _, misses in results) return total_hits, total_misses - def volume(self): """Return estimated total size of cache on disk. @@ -526,7 +511,6 @@ def volume(self): """ return sum(shard.volume() for shard in self._shards) - def close(self): "Close database connection." for shard in self._shards: @@ -535,40 +519,32 @@ def close(self): self._deques.clear() self._indexes.clear() - def __enter__(self): return self - def __exit__(self, *exception): self.close() - def __getstate__(self): return (self._directory, self._count, self.timeout, type(self.disk)) - def __setstate__(self, state): self.__init__(*state) - def __iter__(self): "Iterate keys in cache including expired items." iterators = (iter(shard) for shard in self._shards) return it.chain.from_iterable(iterators) - def __reversed__(self): "Reverse iterate keys in cache including expired items." iterators = (reversed(shard) for shard in reversed(self._shards)) return it.chain.from_iterable(iterators) - def __len__(self): "Count of items in cache including expired items." return sum(len(shard) for shard in self._shards) - def reset(self, key, value=ENOVAL): """Reset `key` and `value` item from Settings table. @@ -597,7 +573,6 @@ def reset(self, key, value=ENOVAL): break return result - def cache(self, name): """Return Cache with given `name` in subdirectory. @@ -627,7 +602,6 @@ def cache(self, name): _caches[name] = temp return temp - def deque(self, name): """Return Deque with given `name` in subdirectory. @@ -657,7 +631,6 @@ def deque(self, name): _deques[name] = deque return deque - def index(self, name): """Return Index with given `name` in subdirectory. diff --git a/diskcache/persistent.py b/diskcache/persistent.py index 6fc072b..ac5a5b0 100644 --- a/diskcache/persistent.py +++ b/diskcache/persistent.py @@ -15,6 +15,7 @@ def _make_compare(seq_op, doc): "Make compare method with Sequence semantics." + def compare(self, that): "Compare method for deque and sequence." if not isinstance(that, Sequence): @@ -70,6 +71,7 @@ class Deque(Sequence): [3, 2, 1, 0, 0, -1, -2, -3] """ + def __init__(self, iterable=(), directory=None): """Initialize deque instance. @@ -83,7 +85,6 @@ def __init__(self, iterable=(), directory=None): self._cache = Cache(directory, eviction_policy='none') self.extend(iterable) - @classmethod def fromcache(cls, cache, iterable=()): """Initialize deque using `cache`. @@ -110,19 +111,16 @@ def fromcache(cls, cache, iterable=()): self.extend(iterable) return self - @property def cache(self): "Cache used by deque." return self._cache - @property def directory(self): "Directory path where deque is stored." return self._cache.directory - def _index(self, index, func): len_self = len(self) @@ -153,7 +151,6 @@ def _index(self, index, func): raise IndexError('deque index out of range') - def __getitem__(self, index): """deque.__getitem__(index) <==> deque[index] @@ -176,7 +173,6 @@ def __getitem__(self, index): """ return self._index(index, self._cache.__getitem__) - def __setitem__(self, index, value): """deque.__setitem__(index, value) <==> deque[index] = value @@ -195,10 +191,11 @@ def __setitem__(self, index, value): :raises IndexError: if index out of range """ + def _set_value(key): return self._cache.__setitem__(key, value) - self._index(index, _set_value) + self._index(index, _set_value) def __delitem__(self, index): """deque.__delitem__(index) <==> del deque[index] @@ -219,7 +216,6 @@ def __delitem__(self, index): """ self._index(index, self._cache.__delitem__) - def __repr__(self): """deque.__repr__() <==> repr(deque) @@ -229,7 +225,6 @@ def __repr__(self): name = type(self).__name__ return '{0}(directory={1!r})'.format(name, self.directory) - __eq__ = _make_compare(op.eq, 'equal to') __ne__ = _make_compare(op.ne, 'not equal to') __lt__ = _make_compare(op.lt, 'less than') @@ -237,7 +232,6 @@ def __repr__(self): __le__ = _make_compare(op.le, 'less than or equal to') __ge__ = _make_compare(op.ge, 'greater than or equal to') - def __iadd__(self, iterable): """deque.__iadd__(iterable) <==> deque += iterable @@ -250,7 +244,6 @@ def __iadd__(self, iterable): self.extend(iterable) return self - def __iter__(self): """deque.__iter__() <==> iter(deque) @@ -265,7 +258,6 @@ def __iter__(self): except KeyError: pass - def __len__(self): """deque.__len__() <==> len(deque) @@ -274,7 +266,6 @@ def __len__(self): """ return len(self._cache) - def __reversed__(self): """deque.__reversed__() <==> reversed(deque) @@ -297,15 +288,12 @@ def __reversed__(self): except KeyError: pass - def __getstate__(self): return self.directory - def __setstate__(self, state): self.__init__(directory=state) - def append(self, value): """Add `value` to back of deque. @@ -321,7 +309,6 @@ def append(self, value): """ self._cache.push(value, retry=True) - def appendleft(self, value): """Add `value` to front of deque. @@ -337,7 +324,6 @@ def appendleft(self, value): """ self._cache.push(value, side='front', retry=True) - def clear(self): """Remove all elements from deque. @@ -351,7 +337,6 @@ def clear(self): """ self._cache.clear(retry=True) - def count(self, value): """Return number of occurrences of `value` in deque. @@ -370,7 +355,6 @@ def count(self, value): """ return sum(1 for item in self if value == item) - def extend(self, iterable): """Extend back side of deque with values from `iterable`. @@ -380,7 +364,6 @@ def extend(self, iterable): for value in iterable: self.append(value) - def extendleft(self, iterable): """Extend front side of deque with value from `iterable`. @@ -395,7 +378,6 @@ def extendleft(self, iterable): for value in iterable: self.appendleft(value) - def peek(self): """Peek at value at back of deque. @@ -422,7 +404,6 @@ def peek(self): raise IndexError('peek from an empty deque') return value - def peekleft(self): """Peek at value at front of deque. @@ -449,7 +430,6 @@ def peekleft(self): raise IndexError('peek from an empty deque') return value - def pop(self): """Remove and return value at back of deque. @@ -476,7 +456,6 @@ def pop(self): raise IndexError('pop from an empty deque') return value - def popleft(self): """Remove and return value at front of deque. @@ -501,7 +480,6 @@ def popleft(self): raise IndexError('pop from an empty deque') return value - def remove(self, value): """Remove first occurrence of `value` in deque. @@ -539,7 +517,6 @@ def remove(self, value): raise ValueError('deque.remove(value): value not in deque') - def reverse(self): """Reverse deque in place. @@ -562,7 +539,6 @@ def reverse(self): del temp rmtree(directory) - def rotate(self, steps=1): """Rotate deque right by `steps`. @@ -611,10 +587,8 @@ def rotate(self, steps=1): else: self.append(value) - __hash__ = None - @contextmanager def transact(self): """Context manager to perform a transaction by locking the deque. @@ -664,6 +638,7 @@ class Index(MutableMapping): ('c', 3) """ + def __init__(self, *args, **kwargs): """Initialize index in directory and update items. @@ -691,7 +666,6 @@ def __init__(self, *args, **kwargs): self._cache = Cache(directory, eviction_policy='none') self.update(*args, **kwargs) - @classmethod def fromcache(cls, cache, *args, **kwargs): """Initialize index using `cache` and update items. @@ -719,19 +693,16 @@ def fromcache(cls, cache, *args, **kwargs): self.update(*args, **kwargs) return self - @property def cache(self): "Cache used by index." return self._cache - @property def directory(self): "Directory path where items are stored." return self._cache.directory - def __getitem__(self, key): """index.__getitem__(key) <==> index[key] @@ -755,7 +726,6 @@ def __getitem__(self, key): """ return self._cache[key] - def __setitem__(self, key, value): """index.__setitem__(key, value) <==> index[key] = value @@ -773,7 +743,6 @@ def __setitem__(self, key, value): """ self._cache[key] = value - def __delitem__(self, key): """index.__delitem__(key) <==> del index[key] @@ -796,7 +765,6 @@ def __delitem__(self, key): """ del self._cache[key] - def setdefault(self, key, default=None): """Set and get value for `key` in index using `default`. @@ -821,7 +789,6 @@ def setdefault(self, key, default=None): except KeyError: _cache.add(key, default, retry=True) - def peekitem(self, last=True): """Peek at key and value item pair in index based on iteration order. @@ -840,7 +807,6 @@ def peekitem(self, last=True): """ return self._cache.peekitem(last, retry=True) - def pop(self, key, default=ENOVAL): """Remove corresponding item for `key` from index and return value. @@ -871,7 +837,6 @@ def pop(self, key, default=ENOVAL): raise KeyError(key) return value - def popitem(self, last=True): """Remove and return item pair. @@ -906,7 +871,6 @@ def popitem(self, last=True): return key, value - def push(self, value, prefix=None, side='back'): """Push `value` onto `side` of queue in index identified by `prefix`. @@ -938,7 +902,6 @@ def push(self, value, prefix=None, side='back'): """ return self._cache.push(value, prefix, side, retry=True) - def pull(self, prefix=None, default=(None, None), side='front'): """Pull key and value item pair from `side` of queue in index. @@ -979,7 +942,6 @@ def pull(self, prefix=None, default=(None, None), side='front'): """ return self._cache.pull(prefix, default, side, retry=True) - def clear(self): """Remove all items from index. @@ -993,7 +955,6 @@ def clear(self): """ self._cache.clear(retry=True) - def __iter__(self): """index.__iter__() <==> iter(index) @@ -1002,7 +963,6 @@ def __iter__(self): """ return iter(self._cache) - def __reversed__(self): """index.__reversed__() <==> reversed(index) @@ -1019,7 +979,6 @@ def __reversed__(self): """ return reversed(self._cache) - def __len__(self): """index.__len__() <==> len(index) @@ -1028,7 +987,6 @@ def __len__(self): """ return len(self._cache) - def keys(self): """Set-like object providing a view of index keys. @@ -1043,7 +1001,6 @@ def keys(self): """ return KeysView(self) - def values(self): """Set-like object providing a view of index values. @@ -1058,7 +1015,6 @@ def values(self): """ return ValuesView(self) - def items(self): """Set-like object providing a view of index items. @@ -1073,18 +1029,14 @@ def items(self): """ return ItemsView(self) - __hash__ = None - def __getstate__(self): return self.directory - def __setstate__(self, state): self.__init__(state) - def __eq__(self, other): """index.__eq__(other) <==> index == other @@ -1118,7 +1070,6 @@ def __eq__(self, other): else: return all(self[key] == other.get(key, ENOVAL) for key in self) - def __ne__(self, other): """index.__ne__(other) <==> index != other @@ -1142,7 +1093,6 @@ def __ne__(self, other): """ return not self == other - def memoize(self, name=None, typed=False): """Memoizing cache decorator. @@ -1199,7 +1149,6 @@ def memoize(self, name=None, typed=False): """ return self._cache.memoize(name, typed) - @contextmanager def transact(self): """Context manager to perform a transaction by locking the index. @@ -1227,7 +1176,6 @@ def transact(self): with self._cache.transact(retry=True): yield - def __repr__(self): """index.__repr__() <==> repr(index) diff --git a/diskcache/recipes.py b/diskcache/recipes.py index 85509c7..3acddd8 100644 --- a/diskcache/recipes.py +++ b/diskcache/recipes.py @@ -32,6 +32,7 @@ class Averager(object): None """ + def __init__(self, cache, key, expire=None, tag=None): self._cache = cache self._key = key @@ -45,7 +46,10 @@ def add(self, value): total += value count += 1 self._cache.set( - self._key, (total, count), expire=self._expire, tag=self._tag, + self._key, + (total, count), + expire=self._expire, + tag=self._tag, ) def get(self): @@ -71,6 +75,7 @@ class Lock(object): ... pass """ + def __init__(self, cache, key, expire=None, tag=None): self._cache = cache self._key = key @@ -124,6 +129,7 @@ class RLock(object): AssertionError: cannot release un-acquired lock """ + def __init__(self, cache, key, expire=None, tag=None): self._cache = cache self._key = key @@ -141,8 +147,10 @@ def acquire(self): value, count = self._cache.get(self._key, default=(None, 0)) if pid_tid == value or count == 0: self._cache.set( - self._key, (pid_tid, count + 1), - expire=self._expire, tag=self._tag, + self._key, + (pid_tid, count + 1), + expire=self._expire, + tag=self._tag, ) return time.sleep(0.001) @@ -158,8 +166,10 @@ def release(self): is_owned = pid_tid == value and count > 0 assert is_owned, 'cannot release un-acquired lock' self._cache.set( - self._key, (value, count - 1), - expire=self._expire, tag=self._tag, + self._key, + (value, count - 1), + expire=self._expire, + tag=self._tag, ) def __enter__(self): @@ -187,6 +197,7 @@ class BoundedSemaphore(object): AssertionError: cannot release un-acquired semaphore """ + def __init__(self, cache, key, value=1, expire=None, tag=None): self._cache = cache self._key = key @@ -201,8 +212,10 @@ def acquire(self): value = self._cache.get(self._key, default=self._value) if value > 0: self._cache.set( - self._key, value - 1, - expire=self._expire, tag=self._tag, + self._key, + value - 1, + expire=self._expire, + tag=self._tag, ) return time.sleep(0.001) @@ -214,7 +227,10 @@ def release(self): assert self._value > value, 'cannot release un-acquired semaphore' value += 1 self._cache.set( - self._key, value, expire=self._expire, tag=self._tag, + self._key, + value, + expire=self._expire, + tag=self._tag, ) def __enter__(self): @@ -224,8 +240,16 @@ def __exit__(self, *exc_info): self.release() -def throttle(cache, count, seconds, name=None, expire=None, tag=None, - time_func=time.time, sleep_func=time.sleep): +def throttle( + cache, + count, + seconds, + name=None, + expire=None, + tag=None, + time_func=time.time, + sleep_func=time.sleep, +): """Decorator to throttle calls to function. >>> import diskcache, time @@ -242,6 +266,7 @@ def throttle(cache, count, seconds, name=None, expire=None, tag=None, True """ + def decorator(func): rate = count / float(seconds) key = full_name(func) if name is None else name @@ -298,6 +323,7 @@ def barrier(cache, lock_factory, name=None, expire=None, tag=None): >>> pool.terminate() """ + def decorator(func): key = full_name(func) if name is None else name lock = lock_factory(cache, key, expire=expire, tag=tag) @@ -385,7 +411,10 @@ def wrapper(*args, **kwargs): "Wrapper for callable to cache arguments and return values." key = wrapper.__cache_key__(*args, **kwargs) pair, expire_time = cache.get( - key, default=ENOVAL, expire_time=True, retry=True, + key, + default=ENOVAL, + expire_time=True, + retry=True, ) if pair is not ENOVAL: @@ -400,7 +429,10 @@ def wrapper(*args, **kwargs): thread_key = key + (ENOVAL,) thread_added = cache.add( - thread_key, None, expire=delta, retry=True, + thread_key, + None, + expire=delta, + retry=True, ) if thread_added: @@ -409,8 +441,13 @@ def recompute(): with cache: pair = timer(*args, **kwargs) cache.set( - key, pair, expire=expire, tag=tag, retry=True, + key, + pair, + expire=expire, + tag=tag, + retry=True, ) + thread = threading.Thread(target=recompute) thread.daemon = True thread.start() diff --git a/setup.py b/setup.py index f8c465e..2d6daa3 100644 --- a/setup.py +++ b/setup.py @@ -10,8 +10,10 @@ def finalize_options(self): TestCommand.finalize_options(self) self.test_args = [] self.test_suite = True + def run_tests(self): import tox + errno = tox.cmdline(self.test_args) exit(errno) diff --git a/tests/benchmark_core.py b/tests/benchmark_core.py index 2eeae3b..a048b79 100644 --- a/tests/benchmark_core.py +++ b/tests/benchmark_core.py @@ -32,19 +32,30 @@ import diskcache -caches.append(('diskcache.Cache', diskcache.Cache, ('tmp',), {},)) -caches.append(( - 'diskcache.FanoutCache(shards=4, timeout=1.0)', - diskcache.FanoutCache, - ('tmp',), - {'shards': 4, 'timeout': 1.0} -)) -caches.append(( - 'diskcache.FanoutCache(shards=8, timeout=0.010)', - diskcache.FanoutCache, - ('tmp',), - {'shards': 8, 'timeout': 0.010} -)) +caches.append( + ( + 'diskcache.Cache', + diskcache.Cache, + ('tmp',), + {}, + ) +) +caches.append( + ( + 'diskcache.FanoutCache(shards=4, timeout=1.0)', + diskcache.FanoutCache, + ('tmp',), + {'shards': 4, 'timeout': 1.0}, + ) +) +caches.append( + ( + 'diskcache.FanoutCache(shards=8, timeout=0.010)', + diskcache.FanoutCache, + ('tmp',), + {'shards': 8, 'timeout': 0.010}, + ) +) ############################################################################### @@ -54,12 +65,17 @@ try: import pylibmc - caches.append(( - 'pylibmc.Client', - pylibmc.Client, - (['127.0.0.1'],), - {'binary': True, 'behaviors': {'tcp_nodelay': True, 'ketama': True}}, - )) + caches.append( + ( + 'pylibmc.Client', + pylibmc.Client, + (['127.0.0.1'],), + { + 'binary': True, + 'behaviors': {'tcp_nodelay': True, 'ketama': True}, + }, + ) + ) except ImportError: warnings.warn('skipping pylibmc') @@ -71,12 +87,14 @@ try: import redis - caches.append(( - 'redis.StrictRedis', - redis.StrictRedis, - (), - {'host': 'localhost', 'port': 6379, 'db': 0}, - )) + caches.append( + ( + 'redis.StrictRedis', + redis.StrictRedis, + (), + {'host': 'localhost', 'port': 6379, 'db': 0}, + ) + ) except ImportError: warnings.warn('skipping redis') @@ -84,7 +102,7 @@ def worker(num, kind, args, kwargs): random.seed(num) - time.sleep(0.01) # Let other processes start. + time.sleep(0.01) # Let other processes start. obj = kind(*args, **kwargs) @@ -173,19 +191,31 @@ def dispatch(): formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) parser.add_argument( - '-p', '--processes', type=int, default=PROCS, + '-p', + '--processes', + type=int, + default=PROCS, help='Number of processes to start', ) parser.add_argument( - '-n', '--operations', type=float, default=OPS, + '-n', + '--operations', + type=float, + default=OPS, help='Number of operations to perform', ) parser.add_argument( - '-r', '--range', type=int, default=RANGE, + '-r', + '--range', + type=int, + default=RANGE, help='Range of keys', ) parser.add_argument( - '-w', '--warmup', type=float, default=WARMUP, + '-w', + '--warmup', + type=float, + default=WARMUP, help='Number of warmup operations before timings', ) diff --git a/tests/benchmark_djangocache.py b/tests/benchmark_djangocache.py index 0512752..0de0424 100644 --- a/tests/benchmark_djangocache.py +++ b/tests/benchmark_djangocache.py @@ -27,6 +27,7 @@ def setup(): os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'tests.settings_benchmark') import django + django.setup() @@ -41,7 +42,7 @@ def worker(num, name): timings = co.defaultdict(list) - time.sleep(0.01) # Let other processes start. + time.sleep(0.01) # Let other processes start. for count in range(OPS): key = str(random.randrange(RANGE)).encode('utf-8') @@ -140,19 +141,31 @@ def dispatch(): formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) parser.add_argument( - '-p', '--processes', type=int, default=PROCS, + '-p', + '--processes', + type=int, + default=PROCS, help='Number of processes to start', ) parser.add_argument( - '-n', '--operations', type=float, default=OPS, + '-n', + '--operations', + type=float, + default=OPS, help='Number of operations to perform', ) parser.add_argument( - '-r', '--range', type=int, default=RANGE, + '-r', + '--range', + type=int, + default=RANGE, help='Range of keys', ) parser.add_argument( - '-w', '--warmup', type=float, default=WARMUP, + '-w', + '--warmup', + type=float, + default=WARMUP, help='Number of warmup operations before timings', ) diff --git a/tests/benchmark_glob.py b/tests/benchmark_glob.py index 82237de..9c23104 100644 --- a/tests/benchmark_glob.py +++ b/tests/benchmark_glob.py @@ -26,11 +26,9 @@ for value in range(count): with open(op.join('tmp', '%s.tmp' % value), 'wb') as writer: pass - + delta = timeit.timeit( - stmt="glob.glob1('tmp', '*.tmp')", - setup='import glob', - number=100 + stmt="glob.glob1('tmp', '*.tmp')", setup='import glob', number=100 ) print(template % (count, secs(delta))) diff --git a/tests/issue_109.py b/tests/issue_109.py index a650b4c..a5355b1 100644 --- a/tests/issue_109.py +++ b/tests/issue_109.py @@ -8,6 +8,7 @@ def main(): import argparse + parser = argparse.ArgumentParser() parser.add_argument('--cache-dir', default='/tmp/test') parser.add_argument('--iterations', type=int, default=100) @@ -31,7 +32,7 @@ def main(): delays.append(diff) # Discard warmup delays, first two iterations. - del delays[:(len(values) * 2)] + del delays[: (len(values) * 2)] # Convert seconds to microseconds. delays = sorted(delay * 1e6 for delay in delays) diff --git a/tests/issue_85.py b/tests/issue_85.py index a52de97..db32046 100644 --- a/tests/issue_85.py +++ b/tests/issue_85.py @@ -26,6 +26,7 @@ def init_django(): os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'tests.settings') django.setup() from django.core.cache import cache + shard = cache._cache._shards[0] diff --git a/tests/models.py b/tests/models.py index 349fd87..a546822 100644 --- a/tests/models.py +++ b/tests/models.py @@ -10,4 +10,6 @@ def expensive_calculation(): class Poll(models.Model): question = models.CharField(max_length=200) answer = models.CharField(max_length=200) - pub_date = models.DateTimeField('date published', default=expensive_calculation) + pub_date = models.DateTimeField( + 'date published', default=expensive_calculation + ) diff --git a/tests/plot.py b/tests/plot.py index d8d1be0..2f83f14 100644 --- a/tests/plot.py +++ b/tests/plot.py @@ -93,12 +93,17 @@ def make_plot(data, action, save=False, show=False, limit=0.005): bars = [] for pos, (name, color) in enumerate(zip(names, colors)): - bars.append(ax.bar( - [val + pos * width for val in index], - [parse_timing(data[name][action][tick], limit) for tick in ticks], - width, - color=color, - )) + bars.append( + ax.bar( + [val + pos * width for val in index], + [ + parse_timing(data[name][action][tick], limit) + for tick in ticks + ], + width, + color=color, + ) + ) ax.set_ylabel('Time (microseconds)') ax.set_title('"%s" Time vs Percentile' % action) @@ -106,12 +111,14 @@ def make_plot(data, action, save=False, show=False, limit=0.005): ax.set_xticklabels(ticks) box = ax.get_position() - ax.set_position([box.x0, box.y0 + box.height * 0.2, box.width, box.height * 0.8]) + ax.set_position( + [box.x0, box.y0 + box.height * 0.2, box.width, box.height * 0.8] + ) ax.legend( [bar[0] for bar in bars], names, loc='lower center', - bbox_to_anchor=(0.5, -0.25) + bbox_to_anchor=(0.5, -0.25), ) if show: diff --git a/tests/plot_early_recompute.py b/tests/plot_early_recompute.py index bbebc1a..da46e3e 100644 --- a/tests/plot_early_recompute.py +++ b/tests/plot_early_recompute.py @@ -16,6 +16,7 @@ def make_timer(times): """ lock = threading.Lock() + def timer(func): @ft.wraps(func) def wrapper(*args, **kwargs): @@ -24,7 +25,9 @@ def wrapper(*args, **kwargs): pair = start, time.time() with lock: times.append(pair) + return wrapper + return timer @@ -33,9 +36,11 @@ def make_worker(times, delay=0.2): `delay` seconds. """ + @make_timer(times) def worker(): time.sleep(delay) + return worker @@ -44,11 +49,13 @@ def make_repeater(func, total=10, delay=0.01): repeatedly until `total` seconds have elapsed. """ + def repeat(num): start = time.time() while time.time() - start < total: func() time.sleep(delay) + return repeat @@ -62,6 +69,7 @@ def frange(start, stop, step=1e-3): def plot(option, filename, cache_times, worker_times): "Plot concurrent workers and latency." import matplotlib.pyplot as plt + fig, (workers, latency) = plt.subplots(2, sharex=True) fig.suptitle(option) diff --git a/tests/settings_benchmark.py b/tests/settings_benchmark.py index 5b614a5..1724cef 100644 --- a/tests/settings_benchmark.py +++ b/tests/settings_benchmark.py @@ -14,7 +14,7 @@ 'LOCATION': 'redis://127.0.0.1:6379/1', 'OPTIONS': { 'CLIENT_CLASS': 'django_redis.client.DefaultClient', - } + }, }, 'filebased': { 'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', @@ -22,7 +22,7 @@ 'OPTIONS': { 'CULL_FREQUENCY': 10, 'MAX_ENTRIES': 1000, - } + }, }, 'locmem': { 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', @@ -30,7 +30,7 @@ 'OPTIONS': { 'CULL_FREQUENCY': 10, 'MAX_ENTRIES': 1000, - } + }, }, 'diskcache': { 'BACKEND': 'diskcache.DjangoCache', diff --git a/tests/stress_test_core.py b/tests/stress_test_core.py index 68f2ebd..a36ae4e 100644 --- a/tests/stress_test_core.py +++ b/tests/stress_test_core.py @@ -33,13 +33,17 @@ def make_long(): def make_unicode(): word_size = random.randint(1, 26) - word = u''.join(random.sample(u'abcdefghijklmnopqrstuvwxyz', word_size)) + word = u''.join( + random.sample(u'abcdefghijklmnopqrstuvwxyz', word_size) + ) size = random.randint(1, int(200 / 13)) return word * size def make_bytes(): word_size = random.randint(1, 26) - word = u''.join(random.sample(u'abcdefghijklmnopqrstuvwxyz', word_size)).encode('utf-8') + word = u''.join( + random.sample(u'abcdefghijklmnopqrstuvwxyz', word_size) + ).encode('utf-8') size = random.randint(1, int(200 / 13)) return word * size @@ -49,7 +53,14 @@ def make_float(): def make_object(): return (make_float(),) * random.randint(1, 20) - funcs = [make_int, make_long, make_unicode, make_bytes, make_float, make_object] + funcs = [ + make_int, + make_long, + make_unicode, + make_bytes, + make_float, + make_object, + ] while True: func = random.choice(funcs) @@ -66,13 +77,17 @@ def make_long(): def make_unicode(): word_size = random.randint(1, 26) - word = u''.join(random.sample(u'abcdefghijklmnopqrstuvwxyz', word_size)) + word = u''.join( + random.sample(u'abcdefghijklmnopqrstuvwxyz', word_size) + ) size = random.randint(1, int(2 ** 16 / 13)) return word * size def make_bytes(): word_size = random.randint(1, 26) - word = u''.join(random.sample(u'abcdefghijklmnopqrstuvwxyz', word_size)).encode('utf-8') + word = u''.join( + random.sample(u'abcdefghijklmnopqrstuvwxyz', word_size) + ).encode('utf-8') size = random.randint(1, int(2 ** 16 / 13)) return word * size @@ -82,7 +97,14 @@ def make_float(): def make_object(): return [make_float()] * random.randint(1, int(2e3)) - funcs = [make_int, make_long, make_unicode, make_bytes, make_float, make_object] + funcs = [ + make_int, + make_long, + make_unicode, + make_bytes, + make_float, + make_object, + ] while True: func = random.choice(funcs) @@ -134,7 +156,12 @@ def worker(queue, eviction_policy, processes, threads): stop = time.time() - if action == 'get' and processes == 1 and threads == 1 and EXPIRE is None: + if ( + action == 'get' + and processes == 1 + and threads == 1 + and EXPIRE is None + ): assert result == value if index > WARMUP: @@ -155,8 +182,10 @@ def dispatch(num, eviction_policy, processes, threads): thread_queues = [queue.Queue() for _ in range(threads)] subthreads = [ threading.Thread( - target=worker, args=(thread_queue, eviction_policy, processes, threads) - ) for thread_queue in thread_queues + target=worker, + args=(thread_queue, eviction_policy, processes, threads), + ) + for thread_queue in thread_queues ] for index, triplet in enumerate(process_queue): @@ -201,9 +230,13 @@ def percentile(sequence, percent): return values[pos] -def stress_test(create=True, delete=True, - eviction_policy=u'least-recently-stored', - processes=1, threads=1): +def stress_test( + create=True, + delete=True, + eviction_policy=u'least-recently-stored', + processes=1, + threads=1, +): shutil.rmtree('tmp', ignore_errors=True) if processes == 1: @@ -285,51 +318,84 @@ def stress_test_mp(): formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) parser.add_argument( - '-n', '--operations', type=float, default=OPERATIONS, + '-n', + '--operations', + type=float, + default=OPERATIONS, help='Number of operations to perform', ) parser.add_argument( - '-g', '--get-average', type=float, default=GET_AVERAGE, + '-g', + '--get-average', + type=float, + default=GET_AVERAGE, help='Expected value of exponential variate used for GET count', ) parser.add_argument( - '-k', '--key-count', type=float, default=KEY_COUNT, - help='Number of unique keys' + '-k', + '--key-count', + type=float, + default=KEY_COUNT, + help='Number of unique keys', ) parser.add_argument( - '-d', '--del-chance', type=float, default=DEL_CHANCE, + '-d', + '--del-chance', + type=float, + default=DEL_CHANCE, help='Likelihood of a key deletion', ) parser.add_argument( - '-w', '--warmup', type=float, default=WARMUP, + '-w', + '--warmup', + type=float, + default=WARMUP, help='Number of warmup operations before timings', ) parser.add_argument( - '-e', '--expire', type=float, default=EXPIRE, + '-e', + '--expire', + type=float, + default=EXPIRE, help='Number of seconds before key expires', ) parser.add_argument( - '-t', '--threads', type=int, default=1, + '-t', + '--threads', + type=int, + default=1, help='Number of threads to start in each process', ) parser.add_argument( - '-p', '--processes', type=int, default=1, + '-p', + '--processes', + type=int, + default=1, help='Number of processes to start', ) parser.add_argument( - '-s', '--seed', type=int, default=0, + '-s', + '--seed', + type=int, + default=0, help='Random seed', ) parser.add_argument( - '--no-create', action='store_false', dest='create', + '--no-create', + action='store_false', + dest='create', help='Do not create operations data', ) parser.add_argument( - '--no-delete', action='store_false', dest='delete', + '--no-delete', + action='store_false', + dest='delete', help='Do not delete operations data', ) parser.add_argument( - '-v', '--eviction-policy', type=unicode, + '-v', + '--eviction-policy', + type=unicode, default=u'least-recently-stored', ) diff --git a/tests/stress_test_fanout.py b/tests/stress_test_fanout.py index 422e874..fc2de39 100644 --- a/tests/stress_test_fanout.py +++ b/tests/stress_test_fanout.py @@ -33,13 +33,17 @@ def make_long(): def make_unicode(): word_size = random.randint(1, 26) - word = u''.join(random.sample(u'abcdefghijklmnopqrstuvwxyz', word_size)) + word = u''.join( + random.sample(u'abcdefghijklmnopqrstuvwxyz', word_size) + ) size = random.randint(1, int(200 / 13)) return word * size def make_bytes(): word_size = random.randint(1, 26) - word = u''.join(random.sample(u'abcdefghijklmnopqrstuvwxyz', word_size)).encode('utf-8') + word = u''.join( + random.sample(u'abcdefghijklmnopqrstuvwxyz', word_size) + ).encode('utf-8') size = random.randint(1, int(200 / 13)) return word * size @@ -49,7 +53,14 @@ def make_float(): def make_object(): return (make_float(),) * random.randint(1, 20) - funcs = [make_int, make_long, make_unicode, make_bytes, make_float, make_object] + funcs = [ + make_int, + make_long, + make_unicode, + make_bytes, + make_float, + make_object, + ] while True: func = random.choice(funcs) @@ -66,13 +77,17 @@ def make_long(): def make_unicode(): word_size = random.randint(1, 26) - word = u''.join(random.sample(u'abcdefghijklmnopqrstuvwxyz', word_size)) + word = u''.join( + random.sample(u'abcdefghijklmnopqrstuvwxyz', word_size) + ) size = random.randint(1, int(2 ** 16 / 13)) return word * size def make_bytes(): word_size = random.randint(1, 26) - word = u''.join(random.sample(u'abcdefghijklmnopqrstuvwxyz', word_size)).encode('utf-8') + word = u''.join( + random.sample(u'abcdefghijklmnopqrstuvwxyz', word_size) + ).encode('utf-8') size = random.randint(1, int(2 ** 16 / 13)) return word * size @@ -82,7 +97,14 @@ def make_float(): def make_object(): return [make_float()] * random.randint(1, int(2e3)) - funcs = [make_int, make_long, make_unicode, make_bytes, make_float, make_object] + funcs = [ + make_int, + make_long, + make_unicode, + make_bytes, + make_float, + make_object, + ] while True: func = random.choice(funcs) @@ -129,7 +151,12 @@ def worker(queue, eviction_policy, processes, threads): stop = time.time() - if action == 'get' and processes == 1 and threads == 1 and EXPIRE is None: + if ( + action == 'get' + and processes == 1 + and threads == 1 + and EXPIRE is None + ): assert result == value if index > WARMUP: @@ -147,8 +174,10 @@ def dispatch(num, eviction_policy, processes, threads): thread_queues = [queue.Queue() for _ in range(threads)] subthreads = [ threading.Thread( - target=worker, args=(thread_queue, eviction_policy, processes, threads) - ) for thread_queue in thread_queues + target=worker, + args=(thread_queue, eviction_policy, processes, threads), + ) + for thread_queue in thread_queues ] for index, triplet in enumerate(process_queue): @@ -193,9 +222,13 @@ def percentile(sequence, percent): return values[pos] -def stress_test(create=True, delete=True, - eviction_policy=u'least-recently-stored', - processes=1, threads=1): +def stress_test( + create=True, + delete=True, + eviction_policy=u'least-recently-stored', + processes=1, + threads=1, +): shutil.rmtree('tmp', ignore_errors=True) if processes == 1: @@ -277,51 +310,84 @@ def stress_test_mp(): formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) parser.add_argument( - '-n', '--operations', type=float, default=OPERATIONS, + '-n', + '--operations', + type=float, + default=OPERATIONS, help='Number of operations to perform', ) parser.add_argument( - '-g', '--get-average', type=float, default=GET_AVERAGE, + '-g', + '--get-average', + type=float, + default=GET_AVERAGE, help='Expected value of exponential variate used for GET count', ) parser.add_argument( - '-k', '--key-count', type=float, default=KEY_COUNT, - help='Number of unique keys' + '-k', + '--key-count', + type=float, + default=KEY_COUNT, + help='Number of unique keys', ) parser.add_argument( - '-d', '--del-chance', type=float, default=DEL_CHANCE, + '-d', + '--del-chance', + type=float, + default=DEL_CHANCE, help='Likelihood of a key deletion', ) parser.add_argument( - '-w', '--warmup', type=float, default=WARMUP, + '-w', + '--warmup', + type=float, + default=WARMUP, help='Number of warmup operations before timings', ) parser.add_argument( - '-e', '--expire', type=float, default=EXPIRE, + '-e', + '--expire', + type=float, + default=EXPIRE, help='Number of seconds before key expires', ) parser.add_argument( - '-t', '--threads', type=int, default=1, + '-t', + '--threads', + type=int, + default=1, help='Number of threads to start in each process', ) parser.add_argument( - '-p', '--processes', type=int, default=1, + '-p', + '--processes', + type=int, + default=1, help='Number of processes to start', ) parser.add_argument( - '-s', '--seed', type=int, default=0, + '-s', + '--seed', + type=int, + default=0, help='Random seed', ) parser.add_argument( - '--no-create', action='store_false', dest='create', + '--no-create', + action='store_false', + dest='create', help='Do not create operations data', ) parser.add_argument( - '--no-delete', action='store_false', dest='delete', + '--no-delete', + action='store_false', + dest='delete', help='Do not delete operations data', ) parser.add_argument( - '-v', '--eviction-policy', type=unicode, + '-v', + '--eviction-policy', + type=unicode, default=u'least-recently-stored', ) diff --git a/tests/test_core.py b/tests/test_core.py index 31b4034..cb4e34f 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -26,6 +26,7 @@ pytestmark = pytest.mark.filterwarnings('ignore', category=dc.EmptyDirWarning) + @pytest.fixture def cache(): with dc.Cache() as cache: @@ -161,6 +162,7 @@ def test_close_error(cache): class LocalTest(object): def __init__(self): self._calls = 0 + def __getattr__(self, name): if self._calls: raise AttributeError @@ -303,6 +305,7 @@ def test_get(cache): assert cache.get(0, tag=True) == (0, u'number') assert cache.get(0, expire_time=True, tag=True) == (0, None, u'number') + def test_get_expired_fast_path(cache): assert cache.set(0, 0, expire=0.001) time.sleep(0.01) @@ -642,7 +645,7 @@ def test_check(cache): cache.check() cache.check(fix=True) - assert len(cache.check()) == 0 # Should display no warnings. + assert len(cache.check()) == 0 # Should display no warnings. def test_integrity_check(cache): @@ -653,7 +656,7 @@ def test_integrity_check(cache): with io.open(op.join(cache.directory, 'cache.db'), 'r+b') as writer: writer.seek(52) - writer.write(b'\x00\x01') # Should be 0, change it. + writer.write(b'\x00\x01') # Should be 0, change it. cache = dc.Cache(cache.directory) @@ -1262,8 +1265,8 @@ def test_cull_timeout(cache): def test_key_roundtrip(cache): - key_part_0 = u"part0" - key_part_1 = u"part1" + key_part_0 = u'part0' + key_part_1 = u'part1' to_test = [ (key_part_0, key_part_1), [key_part_0, key_part_1], @@ -1281,6 +1284,7 @@ def test_key_roundtrip(cache): def test_constant(): import diskcache.core + assert repr(diskcache.core.ENOVAL) == 'ENOVAL' diff --git a/tests/test_djangocache.py b/tests/test_djangocache.py index 84d2f95..5cb0047 100644 --- a/tests/test_djangocache.py +++ b/tests/test_djangocache.py @@ -20,28 +20,43 @@ from django.conf import settings from django.core import management, signals from django.core.cache import ( - DEFAULT_CACHE_ALIAS, CacheKeyWarning, InvalidCacheKey, cache, caches, + DEFAULT_CACHE_ALIAS, + CacheKeyWarning, + InvalidCacheKey, + cache, + caches, ) from django.core.cache.utils import make_template_fragment_key from django.db import close_old_connections, connection, connections from django.http import ( - HttpRequest, HttpResponse, HttpResponseNotModified, StreamingHttpResponse, + HttpRequest, + HttpResponse, + HttpResponseNotModified, + StreamingHttpResponse, ) from django.middleware.cache import ( - CacheMiddleware, FetchFromCacheMiddleware, UpdateCacheMiddleware, + CacheMiddleware, + FetchFromCacheMiddleware, + UpdateCacheMiddleware, ) from django.middleware.csrf import CsrfViewMiddleware from django.template import engines from django.template.context_processors import csrf from django.template.response import TemplateResponse from django.test import ( - RequestFactory, SimpleTestCase, TestCase, TransactionTestCase, + RequestFactory, + SimpleTestCase, + TestCase, + TransactionTestCase, override_settings, ) from django.test.signals import setting_changed from django.utils import timezone, translation from django.utils.cache import ( - get_cache_key, learn_cache_key, patch_cache_control, patch_vary_headers, + get_cache_key, + learn_cache_key, + patch_cache_control, + patch_vary_headers, ) from django.utils.encoding import force_text from django.views.decorators.cache import cache_page @@ -53,6 +68,7 @@ os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'tests.settings') import django + django.setup() from .models import Poll, expensive_calculation @@ -81,7 +97,7 @@ def __getstate__(self): class UnpicklableType(object): # Unpicklable using the default pickling protocol on Python 2. - __slots__ = 'a', + __slots__ = ('a',) def custom_key_func(key, key_prefix, version): @@ -110,7 +126,9 @@ def caches_setting_for_tests(base=None, exclude=None, **params): # params -> _caches_setting_base -> base base = base or {} exclude = exclude or set() - setting = {k: base.copy() for k in _caches_setting_base if k not in exclude} + setting = { + k: base.copy() for k in _caches_setting_base if k not in exclude + } for key, cache_params in setting.items(): cache_params.update(_caches_setting_base[key]) cache_params.update(params) @@ -126,15 +144,15 @@ def tearDown(self): def test_simple(self): # Simple cache set/get works - cache.set("key", "value") - self.assertEqual(cache.get("key"), "value") + cache.set('key', 'value') + self.assertEqual(cache.get('key'), 'value') def test_add(self): # A key can be added to a cache - cache.add("addkey1", "value") - result = cache.add("addkey1", "newvalue") + cache.add('addkey1', 'value') + result = cache.add('addkey1', 'newvalue') self.assertFalse(result) - self.assertEqual(cache.get("addkey1"), "value") + self.assertEqual(cache.get('addkey1'), 'value') def test_prefix(self): # Test for same cache key conflicts between shared backend @@ -150,37 +168,41 @@ def test_prefix(self): def test_non_existent(self): """Nonexistent cache keys return as None/default.""" - self.assertIsNone(cache.get("does_not_exist")) - self.assertEqual(cache.get("does_not_exist", "bang!"), "bang!") + self.assertIsNone(cache.get('does_not_exist')) + self.assertEqual(cache.get('does_not_exist', 'bang!'), 'bang!') def test_get_many(self): # Multiple cache keys can be returned using get_many cache.set_many({'a': 'a', 'b': 'b', 'c': 'c', 'd': 'd'}) - self.assertEqual(cache.get_many(['a', 'c', 'd']), {'a': 'a', 'c': 'c', 'd': 'd'}) + self.assertEqual( + cache.get_many(['a', 'c', 'd']), {'a': 'a', 'c': 'c', 'd': 'd'} + ) self.assertEqual(cache.get_many(['a', 'b', 'e']), {'a': 'a', 'b': 'b'}) - self.assertEqual(cache.get_many(iter(['a', 'b', 'e'])), {'a': 'a', 'b': 'b'}) + self.assertEqual( + cache.get_many(iter(['a', 'b', 'e'])), {'a': 'a', 'b': 'b'} + ) def test_delete(self): # Cache keys can be deleted cache.set_many({'key1': 'spam', 'key2': 'eggs'}) - self.assertEqual(cache.get("key1"), "spam") - cache.delete("key1") - self.assertIsNone(cache.get("key1")) - self.assertEqual(cache.get("key2"), "eggs") + self.assertEqual(cache.get('key1'), 'spam') + cache.delete('key1') + self.assertIsNone(cache.get('key1')) + self.assertEqual(cache.get('key2'), 'eggs') def test_has_key(self): # The cache can be inspected for cache keys - cache.set("hello1", "goodbye1") - self.assertTrue(cache.has_key("hello1")) - self.assertFalse(cache.has_key("goodbye1")) - cache.set("no_expiry", "here", None) - self.assertTrue(cache.has_key("no_expiry")) + cache.set('hello1', 'goodbye1') + self.assertTrue(cache.has_key('hello1')) + self.assertFalse(cache.has_key('goodbye1')) + cache.set('no_expiry', 'here', None) + self.assertTrue(cache.has_key('no_expiry')) def test_in(self): # The in operator can be used to inspect cache contents - cache.set("hello2", "goodbye2") - self.assertIn("hello2", cache) - self.assertNotIn("goodbye2", cache) + cache.set('hello2', 'goodbye2') + self.assertIn('hello2', cache) + self.assertNotIn('goodbye2', cache) def test_incr(self): # Cache values can be incremented @@ -219,14 +241,14 @@ def test_data_types(self): 'function': f, 'class': C, } - cache.set("stuff", stuff) - self.assertEqual(cache.get("stuff"), stuff) + cache.set('stuff', stuff) + self.assertEqual(cache.get('stuff'), stuff) def test_cache_read_for_model_instance(self): # Don't want fields with callable as default to be called on cache read expensive_calculation.num_runs = 0 Poll.objects.all().delete() - my_poll = Poll.objects.create(question="Well?") + my_poll = Poll.objects.create(question='Well?') self.assertEqual(Poll.objects.count(), 1) pub_date = my_poll.pub_date cache.set('question', my_poll) @@ -239,7 +261,7 @@ def test_cache_write_for_model_instance_with_deferred(self): # Don't want fields with callable as default to be called on cache write expensive_calculation.num_runs = 0 Poll.objects.all().delete() - Poll.objects.create(question="What?") + Poll.objects.create(question='What?') self.assertEqual(expensive_calculation.num_runs, 1) defer_qs = Poll.objects.all().defer('question') self.assertEqual(defer_qs.count(), 1) @@ -252,7 +274,7 @@ def test_cache_read_for_model_instance_with_deferred(self): # Don't want fields with callable as default to be called on cache read expensive_calculation.num_runs = 0 Poll.objects.all().delete() - Poll.objects.create(question="What?") + Poll.objects.create(question='What?') self.assertEqual(expensive_calculation.num_runs, 1) defer_qs = Poll.objects.all().defer('question') self.assertEqual(defer_qs.count(), 1) @@ -261,7 +283,9 @@ def test_cache_read_for_model_instance_with_deferred(self): runs_before_cache_read = expensive_calculation.num_runs cache.get('deferred_queryset') # We only want the default expensive calculation run on creation and set - self.assertEqual(expensive_calculation.num_runs, runs_before_cache_read) + self.assertEqual( + expensive_calculation.num_runs, runs_before_cache_read + ) def test_expiration(self): # Cache values can be set to expire @@ -270,11 +294,11 @@ def test_expiration(self): cache.set('expire3', 'very quickly', 1) time.sleep(2) - self.assertIsNone(cache.get("expire1")) + self.assertIsNone(cache.get('expire1')) - cache.add("expire2", "newvalue") - self.assertEqual(cache.get("expire2"), "newvalue") - self.assertFalse(cache.has_key("expire3")) + cache.add('expire2', 'newvalue') + self.assertEqual(cache.get('expire2'), 'newvalue') + self.assertFalse(cache.has_key('expire3')) def test_touch(self): # cache.touch() updates the timeout. @@ -299,7 +323,7 @@ def test_unicode(self): 'ascii': 'ascii_value', 'unicode_ascii': 'Iñtërnâtiônàlizætiøn1', 'Iñtërnâtiônàlizætiøn': 'Iñtërnâtiônàlizætiøn2', - 'ascii2': {'x': 1} + 'ascii2': {'x': 1}, } # Test `set` for (key, value) in stuff.items(): @@ -325,6 +349,7 @@ def test_unicode(self): def test_binary_string(self): # Binary strings should be cacheable from zlib import compress, decompress + value = 'value_to_be_compressed' compressed_value = compress(value.encode()) @@ -348,9 +373,9 @@ def test_binary_string(self): def test_set_many(self): # Multiple keys can be set using set_many - cache.set_many({"key1": "spam", "key2": "eggs"}) - self.assertEqual(cache.get("key1"), "spam") - self.assertEqual(cache.get("key2"), "eggs") + cache.set_many({'key1': 'spam', 'key2': 'eggs'}) + self.assertEqual(cache.get('key1'), 'spam') + self.assertEqual(cache.get('key2'), 'eggs') def test_set_many_returns_empty_list_on_success(self): """set_many() returns an empty list when all keys are inserted.""" @@ -359,25 +384,25 @@ def test_set_many_returns_empty_list_on_success(self): def test_set_many_expiration(self): # set_many takes a second ``timeout`` parameter - cache.set_many({"key1": "spam", "key2": "eggs"}, 1) + cache.set_many({'key1': 'spam', 'key2': 'eggs'}, 1) time.sleep(2) - self.assertIsNone(cache.get("key1")) - self.assertIsNone(cache.get("key2")) + self.assertIsNone(cache.get('key1')) + self.assertIsNone(cache.get('key2')) def test_delete_many(self): # Multiple keys can be deleted using delete_many cache.set_many({'key1': 'spam', 'key2': 'eggs', 'key3': 'ham'}) - cache.delete_many(["key1", "key2"]) - self.assertIsNone(cache.get("key1")) - self.assertIsNone(cache.get("key2")) - self.assertEqual(cache.get("key3"), "ham") + cache.delete_many(['key1', 'key2']) + self.assertIsNone(cache.get('key1')) + self.assertIsNone(cache.get('key2')) + self.assertEqual(cache.get('key3'), 'ham') def test_clear(self): # The cache can be emptied using clear cache.set_many({'key1': 'spam', 'key2': 'eggs'}) cache.clear() - self.assertIsNone(cache.get("key1")) - self.assertIsNone(cache.get("key2")) + self.assertIsNone(cache.get('key1')) + self.assertIsNone(cache.get('key2')) def test_long_timeout(self): """ @@ -391,7 +416,10 @@ def test_long_timeout(self): cache.add('key2', 'ham', 60 * 60 * 24 * 30 + 1) self.assertEqual(cache.get('key2'), 'ham') - cache.set_many({'key3': 'sausage', 'key4': 'lobster bisque'}, 60 * 60 * 24 * 30 + 1) + cache.set_many( + {'key3': 'sausage', 'key4': 'lobster bisque'}, + 60 * 60 * 24 * 30 + 1, + ) self.assertEqual(cache.get('key3'), 'sausage') self.assertEqual(cache.get('key4'), 'lobster bisque') @@ -437,8 +465,8 @@ def test_zero_timeout(self): def test_float_timeout(self): # Make sure a timeout given as a float doesn't crash anything. - cache.set("key1", "spam", 100.2) - self.assertEqual(cache.get("key1"), "spam") + cache.set('key1', 'spam', 100.2) + self.assertEqual(cache.get('key1'), 'spam') def _perform_cull_test(self, cull_cache, initial_count, final_count): # Create initial cache key entries. This will overflow the cache, @@ -483,7 +511,9 @@ def func(key, *args): def test_invalid_key_characters(self): # memcached doesn't allow whitespace or control characters in keys. key = 'key with spaces and 清' - self._perform_invalid_key_test(key, KEY_ERRORS_WITH_MEMCACHED_MSG % key) + self._perform_invalid_key_test( + key, KEY_ERRORS_WITH_MEMCACHED_MSG % key + ) def test_invalid_key_length(self): # memcached limits key length to 250. @@ -653,43 +683,85 @@ def test_cache_versioning_incr_decr(self): def test_cache_versioning_get_set_many(self): # set, using default version = 1 cache.set_many({'ford1': 37, 'arthur1': 42}) - self.assertEqual(cache.get_many(['ford1', 'arthur1']), {'ford1': 37, 'arthur1': 42}) - self.assertEqual(cache.get_many(['ford1', 'arthur1'], version=1), {'ford1': 37, 'arthur1': 42}) + self.assertEqual( + cache.get_many(['ford1', 'arthur1']), {'ford1': 37, 'arthur1': 42} + ) + self.assertEqual( + cache.get_many(['ford1', 'arthur1'], version=1), + {'ford1': 37, 'arthur1': 42}, + ) self.assertEqual(cache.get_many(['ford1', 'arthur1'], version=2), {}) self.assertEqual(caches['v2'].get_many(['ford1', 'arthur1']), {}) - self.assertEqual(caches['v2'].get_many(['ford1', 'arthur1'], version=1), {'ford1': 37, 'arthur1': 42}) - self.assertEqual(caches['v2'].get_many(['ford1', 'arthur1'], version=2), {}) + self.assertEqual( + caches['v2'].get_many(['ford1', 'arthur1'], version=1), + {'ford1': 37, 'arthur1': 42}, + ) + self.assertEqual( + caches['v2'].get_many(['ford1', 'arthur1'], version=2), {} + ) # set, default version = 1, but manually override version = 2 cache.set_many({'ford2': 37, 'arthur2': 42}, version=2) self.assertEqual(cache.get_many(['ford2', 'arthur2']), {}) self.assertEqual(cache.get_many(['ford2', 'arthur2'], version=1), {}) - self.assertEqual(cache.get_many(['ford2', 'arthur2'], version=2), {'ford2': 37, 'arthur2': 42}) + self.assertEqual( + cache.get_many(['ford2', 'arthur2'], version=2), + {'ford2': 37, 'arthur2': 42}, + ) - self.assertEqual(caches['v2'].get_many(['ford2', 'arthur2']), {'ford2': 37, 'arthur2': 42}) - self.assertEqual(caches['v2'].get_many(['ford2', 'arthur2'], version=1), {}) - self.assertEqual(caches['v2'].get_many(['ford2', 'arthur2'], version=2), {'ford2': 37, 'arthur2': 42}) + self.assertEqual( + caches['v2'].get_many(['ford2', 'arthur2']), + {'ford2': 37, 'arthur2': 42}, + ) + self.assertEqual( + caches['v2'].get_many(['ford2', 'arthur2'], version=1), {} + ) + self.assertEqual( + caches['v2'].get_many(['ford2', 'arthur2'], version=2), + {'ford2': 37, 'arthur2': 42}, + ) # v2 set, using default version = 2 caches['v2'].set_many({'ford3': 37, 'arthur3': 42}) self.assertEqual(cache.get_many(['ford3', 'arthur3']), {}) self.assertEqual(cache.get_many(['ford3', 'arthur3'], version=1), {}) - self.assertEqual(cache.get_many(['ford3', 'arthur3'], version=2), {'ford3': 37, 'arthur3': 42}) + self.assertEqual( + cache.get_many(['ford3', 'arthur3'], version=2), + {'ford3': 37, 'arthur3': 42}, + ) - self.assertEqual(caches['v2'].get_many(['ford3', 'arthur3']), {'ford3': 37, 'arthur3': 42}) - self.assertEqual(caches['v2'].get_many(['ford3', 'arthur3'], version=1), {}) - self.assertEqual(caches['v2'].get_many(['ford3', 'arthur3'], version=2), {'ford3': 37, 'arthur3': 42}) + self.assertEqual( + caches['v2'].get_many(['ford3', 'arthur3']), + {'ford3': 37, 'arthur3': 42}, + ) + self.assertEqual( + caches['v2'].get_many(['ford3', 'arthur3'], version=1), {} + ) + self.assertEqual( + caches['v2'].get_many(['ford3', 'arthur3'], version=2), + {'ford3': 37, 'arthur3': 42}, + ) # v2 set, default version = 2, but manually override version = 1 caches['v2'].set_many({'ford4': 37, 'arthur4': 42}, version=1) - self.assertEqual(cache.get_many(['ford4', 'arthur4']), {'ford4': 37, 'arthur4': 42}) - self.assertEqual(cache.get_many(['ford4', 'arthur4'], version=1), {'ford4': 37, 'arthur4': 42}) + self.assertEqual( + cache.get_many(['ford4', 'arthur4']), {'ford4': 37, 'arthur4': 42} + ) + self.assertEqual( + cache.get_many(['ford4', 'arthur4'], version=1), + {'ford4': 37, 'arthur4': 42}, + ) self.assertEqual(cache.get_many(['ford4', 'arthur4'], version=2), {}) self.assertEqual(caches['v2'].get_many(['ford4', 'arthur4']), {}) - self.assertEqual(caches['v2'].get_many(['ford4', 'arthur4'], version=1), {'ford4': 37, 'arthur4': 42}) - self.assertEqual(caches['v2'].get_many(['ford4', 'arthur4'], version=2), {}) + self.assertEqual( + caches['v2'].get_many(['ford4', 'arthur4'], version=1), + {'ford4': 37, 'arthur4': 42}, + ) + self.assertEqual( + caches['v2'].get_many(['ford4', 'arthur4'], version=2), {} + ) def test_incr_version(self): cache.set('answer', 42, version=2) @@ -825,7 +897,9 @@ def test_get_or_set_version(self): self.assertIsNone(cache.get('brian', version=3)) def test_get_or_set_racing(self): - with mock.patch('%s.%s' % (settings.CACHES['default']['BACKEND'], 'add')) as cache_add: + with mock.patch( + '%s.%s' % (settings.CACHES['default']['BACKEND'], 'add') + ) as cache_add: # Simulate cache.add() failing to add a value. In that case, the # default value should be returned. cache_add.return_value = False @@ -833,7 +907,6 @@ def test_get_or_set_racing(self): class PicklingSideEffect: - def __init__(self, cache): self.cache = cache self.locked = False @@ -843,11 +916,14 @@ def __getstate__(self): return {} -@override_settings(CACHES=caches_setting_for_tests( - BACKEND='diskcache.DjangoCache', -)) +@override_settings( + CACHES=caches_setting_for_tests( + BACKEND='diskcache.DjangoCache', + ) +) class DiskCacheTests(BaseCacheTests, TestCase): "Specific test cases for diskcache.DjangoCache." + def setUp(self): super().setUp() self.dirname = tempfile.mkdtemp() @@ -868,14 +944,18 @@ def test_ignores_non_cache_files(self): with open(fname, 'w'): os.utime(fname, None) cache.clear() - self.assertTrue(os.path.exists(fname), - 'Expected cache.clear to ignore non cache files') + self.assertTrue( + os.path.exists(fname), + 'Expected cache.clear to ignore non cache files', + ) os.remove(fname) def test_clear_does_not_remove_cache_dir(self): cache.clear() - self.assertTrue(os.path.exists(self.dirname), - 'Expected cache.clear to keep the cache dir') + self.assertTrue( + os.path.exists(self.dirname), + 'Expected cache.clear to keep the cache dir', + ) def test_cache_write_unpicklable_type(self): # This fails if not using the highest pickling protocol on Python 2. @@ -942,7 +1022,9 @@ def test_pop(self): self.assertEqual(cache.pop(0, default=1), 1) self.assertEqual(cache.pop(1, expire_time=True), (1, None)) self.assertEqual(cache.pop(2, tag=True), (2, None)) - self.assertEqual(cache.pop(3, expire_time=True, tag=True), (3, None, None)) + self.assertEqual( + cache.pop(3, expire_time=True, tag=True), (3, None, None) + ) self.assertEqual(cache.pop(4, retry=False), 4) def test_pickle(self): @@ -975,6 +1057,7 @@ def test_index(self): def test_memoize(self): with self.assertRaises(TypeError): + @cache.memoize # <-- Missing parens! def test(): pass diff --git a/tests/test_recipes.py b/tests/test_recipes.py index b26a239..2d53d49 100644 --- a/tests/test_recipes.py +++ b/tests/test_recipes.py @@ -31,11 +31,13 @@ def test_averager(cache): def test_rlock(cache): state = {'num': 0} rlock = dc.RLock(cache, 'demo') + def worker(): state['num'] += 1 with rlock: state['num'] += 1 time.sleep(0.1) + with rlock: thread = threading.Thread(target=worker) thread.start() @@ -48,11 +50,13 @@ def worker(): def test_semaphore(cache): state = {'num': 0} semaphore = dc.BoundedSemaphore(cache, 'demo', value=3) + def worker(): state['num'] += 1 with semaphore: state['num'] += 1 time.sleep(0.1) + semaphore.acquire() semaphore.acquire() with semaphore: @@ -68,11 +72,13 @@ def worker(): def test_memoize_stampede(cache): state = {'num': 0} + @dc.memoize_stampede(cache, 0.1) def worker(num): time.sleep(0.01) state['num'] += 1 return num + start = time.time() while (time.time() - start) < 1: worker(100) diff --git a/tests/utils.py b/tests/utils.py index 47da791..504b930 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -96,16 +96,19 @@ def display(name, timings): len_total += len(values) sum_total += sum(values) - print(template % ( - action, - len(values), - len(timings.get(action + '-miss', [])), - secs(percentile(values, 0.5)), - secs(percentile(values, 0.9)), - secs(percentile(values, 0.99)), - secs(percentile(values, 1.0)), - secs(sum(values)), - )) + print( + template + % ( + action, + len(values), + len(timings.get(action + '-miss', [])), + secs(percentile(values, 0.5)), + secs(percentile(values, 0.9)), + secs(percentile(values, 0.99)), + secs(percentile(values, 1.0)), + secs(sum(values)), + ) + ) totals = ('Total', len_total, '', '', '', '', '', secs(sum_total)) print(template % totals) From 3b87dde40b4d0b2a7f96534110b5103ce7c9e1fc Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Fri, 22 Jan 2021 11:27:24 -0800 Subject: [PATCH 029/126] Make imports consistent with isort --- diskcache/__init__.py | 18 ++++++++++++++---- diskcache/djangocache.py | 1 + diskcache/fanout.py | 2 +- diskcache/persistent.py | 12 ++++++++---- setup.py | 1 + tests/benchmark_kv_store.py | 4 ++-- tests/issue_109.py | 1 + tests/issue_85.py | 3 ++- tests/plot.py | 3 ++- tests/plot_early_recompute.py | 3 ++- tests/stress_test_core.py | 3 ++- tests/stress_test_fanout.py | 3 ++- tests/test_core.py | 4 ++-- tests/test_deque.py | 4 ++-- tests/test_djangocache.py | 1 - tests/test_fanout.py | 4 ++-- tests/test_index.py | 4 ++-- tests/test_recipes.py | 7 ++++--- 18 files changed, 50 insertions(+), 28 deletions(-) diff --git a/diskcache/__init__.py b/diskcache/__init__.py index 0300123..befed76 100644 --- a/diskcache/__init__.py +++ b/diskcache/__init__.py @@ -7,18 +7,28 @@ """ from .core import ( + DEFAULT_SETTINGS, + ENOVAL, + EVICTION_POLICY, + UNKNOWN, Cache, Disk, EmptyDirWarning, JSONDisk, - UnknownFileWarning, Timeout, + UnknownFileWarning, ) -from .core import DEFAULT_SETTINGS, ENOVAL, EVICTION_POLICY, UNKNOWN from .fanout import FanoutCache from .persistent import Deque, Index -from .recipes import Averager, BoundedSemaphore, Lock, RLock -from .recipes import barrier, memoize_stampede, throttle +from .recipes import ( + Averager, + BoundedSemaphore, + Lock, + RLock, + barrier, + memoize_stampede, + throttle, +) __all__ = [ 'Averager', diff --git a/diskcache/djangocache.py b/diskcache/djangocache.py index 2f1db07..44f673d 100644 --- a/diskcache/djangocache.py +++ b/diskcache/djangocache.py @@ -1,6 +1,7 @@ "Django-compatible disk and file backed cache." from functools import wraps + from django.core.cache.backends.base import BaseCache try: diff --git a/diskcache/fanout.py b/diskcache/fanout.py index cc40c07..e7987b4 100644 --- a/diskcache/fanout.py +++ b/diskcache/fanout.py @@ -9,7 +9,7 @@ import tempfile import time -from .core import ENOVAL, DEFAULT_SETTINGS, Cache, Disk, Timeout +from .core import DEFAULT_SETTINGS, ENOVAL, Cache, Disk, Timeout from .persistent import Deque, Index diff --git a/diskcache/persistent.py b/diskcache/persistent.py index ac5a5b0..5ff1939 100644 --- a/diskcache/persistent.py +++ b/diskcache/persistent.py @@ -3,14 +3,18 @@ """ import operator as op - from collections import OrderedDict -from collections.abc import MutableMapping, Sequence -from collections.abc import KeysView, ValuesView, ItemsView +from collections.abc import ( + ItemsView, + KeysView, + MutableMapping, + Sequence, + ValuesView, +) from contextlib import contextmanager from shutil import rmtree -from .core import Cache, ENOVAL +from .core import ENOVAL, Cache def _make_compare(seq_op, doc): diff --git a/setup.py b/setup.py index 2d6daa3..5f3cd88 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,5 @@ from io import open + from setuptools import setup from setuptools.command.test import test as TestCommand diff --git a/tests/benchmark_kv_store.py b/tests/benchmark_kv_store.py index e214f1a..e141a36 100644 --- a/tests/benchmark_kv_store.py +++ b/tests/benchmark_kv_store.py @@ -4,10 +4,10 @@ """ -import diskcache - from IPython import get_ipython +import diskcache + ipython = get_ipython() assert ipython is not None, 'No IPython! Run with $ ipython ...' diff --git a/tests/issue_109.py b/tests/issue_109.py index a5355b1..c10a81f 100644 --- a/tests/issue_109.py +++ b/tests/issue_109.py @@ -3,6 +3,7 @@ """ import time + import diskcache as dc diff --git a/tests/issue_85.py b/tests/issue_85.py index db32046..b325df0 100644 --- a/tests/issue_85.py +++ b/tests/issue_85.py @@ -6,7 +6,6 @@ """ import collections -import django import os import random import shutil @@ -14,6 +13,8 @@ import threading import time +import django + def remove_cache_dir(): print('REMOVING CACHE DIRECTORY') diff --git a/tests/plot.py b/tests/plot.py index 2f83f14..670bf49 100644 --- a/tests/plot.py +++ b/tests/plot.py @@ -7,10 +7,11 @@ import argparse import collections as co -import matplotlib.pyplot as plt import re import sys +import matplotlib.pyplot as plt + def parse_timing(timing, limit): "Parse timing." diff --git a/tests/plot_early_recompute.py b/tests/plot_early_recompute.py index da46e3e..7096ea0 100644 --- a/tests/plot_early_recompute.py +++ b/tests/plot_early_recompute.py @@ -2,13 +2,14 @@ """ -import diskcache as dc import functools as ft import multiprocessing.pool import shutil import threading import time +import diskcache as dc + def make_timer(times): """Make a decorator which accumulates (start, end) in `times` for function diff --git a/tests/stress_test_core.py b/tests/stress_test_core.py index a36ae4e..6c48d5c 100644 --- a/tests/stress_test_core.py +++ b/tests/stress_test_core.py @@ -1,7 +1,6 @@ "Stress test diskcache.core.Cache." import collections as co -from diskcache import Cache, UnknownFileWarning, EmptyDirWarning, Timeout import multiprocessing as mp import os import pickle @@ -13,6 +12,8 @@ import time import warnings +from diskcache import Cache, EmptyDirWarning, Timeout, UnknownFileWarning + from .utils import display OPERATIONS = int(1e4) diff --git a/tests/stress_test_fanout.py b/tests/stress_test_fanout.py index fc2de39..1f29987 100644 --- a/tests/stress_test_fanout.py +++ b/tests/stress_test_fanout.py @@ -1,7 +1,6 @@ "Stress test diskcache.core.Cache." import collections as co -from diskcache import FanoutCache, UnknownFileWarning, EmptyDirWarning import multiprocessing as mp import os import pickle @@ -13,6 +12,8 @@ import time import warnings +from diskcache import EmptyDirWarning, FanoutCache, UnknownFileWarning + from .utils import display OPERATIONS = int(1e4) diff --git a/tests/test_core.py b/tests/test_core.py index cb4e34f..6fe50c1 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -8,7 +8,6 @@ import os import os.path as op import pickle -import pytest import random import shutil import sqlite3 @@ -19,9 +18,10 @@ import time import unittest import warnings - from unittest import mock +import pytest + import diskcache as dc pytestmark = pytest.mark.filterwarnings('ignore', category=dc.EmptyDirWarning) diff --git a/tests/test_deque.py b/tests/test_deque.py index 1ca966d..bfb9e7d 100644 --- a/tests/test_deque.py +++ b/tests/test_deque.py @@ -2,12 +2,12 @@ import functools as ft import pickle -import pytest import shutil import tempfile - from unittest import mock +import pytest + import diskcache as dc from diskcache.core import ENOVAL diff --git a/tests/test_djangocache.py b/tests/test_djangocache.py index 5cb0047..2c216fe 100644 --- a/tests/test_djangocache.py +++ b/tests/test_djangocache.py @@ -14,7 +14,6 @@ import time import unittest import warnings - from unittest import mock from django.conf import settings diff --git a/tests/test_fanout.py b/tests/test_fanout.py index aa3c833..effe47d 100644 --- a/tests/test_fanout.py +++ b/tests/test_fanout.py @@ -8,7 +8,6 @@ import os import os.path as op import pickle -import pytest import random import shutil import sqlite3 @@ -18,9 +17,10 @@ import threading import time import warnings - from unittest import mock +import pytest + import diskcache as dc warnings.simplefilter('error') diff --git a/tests/test_index.py b/tests/test_index.py index 575558c..8852e99 100644 --- a/tests/test_index.py +++ b/tests/test_index.py @@ -2,13 +2,13 @@ import functools as ft import pickle -import pytest import shutil import sys import tempfile - from unittest import mock +import pytest + import diskcache as dc diff --git a/tests/test_recipes.py b/tests/test_recipes.py index 2d53d49..efaa47c 100644 --- a/tests/test_recipes.py +++ b/tests/test_recipes.py @@ -1,13 +1,14 @@ "Test diskcache.recipes." -import diskcache as dc -import pytest import shutil import threading import time - from unittest import mock +import pytest + +import diskcache as dc + @pytest.fixture def cache(): From c40b57facd5d9d167a645cb55a4b69257ef20299 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Fri, 22 Jan 2021 11:50:33 -0800 Subject: [PATCH 030/126] Increase max attributes to 8 --- .pylintrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pylintrc b/.pylintrc index 3314897..8458ed4 100644 --- a/.pylintrc +++ b/.pylintrc @@ -541,7 +541,7 @@ valid-metaclass-classmethod-first-arg=cls max-args=8 # Maximum number of attributes for a class (see R0902). -max-attributes=7 +max-attributes=8 # Maximum number of boolean expressions in an if statement. max-bool-expr=5 From 58227d18dbdd885fd27ba74a9a34e74fb72222d3 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Fri, 22 Jan 2021 11:50:46 -0800 Subject: [PATCH 031/126] Tell mypy to ignore django --- mypy.ini | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 mypy.ini diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 0000000..053b283 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,4 @@ +[mypy] + +[mypy-django.*] +ignore_missing_imports = True From db76f111e43ebacc392443b861565bfed14e87b7 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Fri, 22 Jan 2021 11:50:58 -0800 Subject: [PATCH 032/126] Remove useless `dataset` target --- README.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/README.rst b/README.rst index 847b2cf..ba2bf3e 100644 --- a/README.rst +++ b/README.rst @@ -314,7 +314,6 @@ Object Relational Mappings (ORM) .. _`Django ORM`: https://docs.djangoproject.com/en/dev/topics/db/ .. _`SQLAlchemy`: https://www.sqlalchemy.org/ .. _`Peewee`: http://docs.peewee-orm.com/ -.. _`dataset`: https://dataset.readthedocs.io/ .. _`SQLObject`: http://sqlobject.org/ .. _`Pony ORM`: https://ponyorm.com/ From b2c0ae421ccdf332698431f1dd29eb6e9936d375 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Fri, 22 Jan 2021 11:51:57 -0800 Subject: [PATCH 033/126] Ignore type errors when setting class attributes --- diskcache/fanout.py | 2 +- diskcache/persistent.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/diskcache/fanout.py b/diskcache/fanout.py index e7987b4..6a13d6e 100644 --- a/diskcache/fanout.py +++ b/diskcache/fanout.py @@ -664,4 +664,4 @@ def index(self, name): return index -FanoutCache.memoize = Cache.memoize +FanoutCache.memoize = Cache.memoize # type: ignore diff --git a/diskcache/persistent.py b/diskcache/persistent.py index 5ff1939..44f9cc7 100644 --- a/diskcache/persistent.py +++ b/diskcache/persistent.py @@ -591,7 +591,7 @@ def rotate(self, steps=1): else: self.append(value) - __hash__ = None + __hash__ = None # type: ignore @contextmanager def transact(self): @@ -1033,7 +1033,7 @@ def items(self): """ return ItemsView(self) - __hash__ = None + __hash__ = None # type: ignore def __getstate__(self): return self.directory From 585f47f1084581e8d2fa0aed8ce0dd8a2e0a9162 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Fri, 22 Jan 2021 11:52:58 -0800 Subject: [PATCH 034/126] Add django to deps for sphinx and pylint --- tox.ini | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 6bbe69c..356a576 100644 --- a/tox.ini +++ b/tox.ini @@ -28,7 +28,9 @@ commands=doc8 docs allowlist_externals=make changedir=docs commands=make html -deps=sphinx +deps= + django==2.2.* + sphinx [testenv:flake8] commands=flake8 {toxinidir}/setup.py {toxinidir}/diskcache {toxinidir}/tests @@ -48,7 +50,9 @@ deps=mypy [testenv:pylint] commands=pylint {toxinidir}/diskcache -deps=pylint +deps= + django==2.2.* + pylint [testenv:rstcheck] commands=rstcheck {toxinidir}/README.rst From 945ee10ca54871032196a0b468ce17d8e0711dcb Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Fri, 22 Jan 2021 11:53:14 -0800 Subject: [PATCH 035/126] Tell doc8 to ignore docs/_build dir --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 356a576..de47de5 100644 --- a/tox.ini +++ b/tox.ini @@ -22,7 +22,7 @@ deps=blue [testenv:doc8] deps=doc8 -commands=doc8 docs +commands=doc8 docs --ignore-path docs/_build [testenv:docs] allowlist_externals=make From 4bbe73fb251830a2b2e7d78c21a67f2ab9b18804 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Fri, 22 Jan 2021 11:53:36 -0800 Subject: [PATCH 036/126] Update flake8 configs --- tox.ini | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tox.ini b/tox.ini index de47de5..d5f43c0 100644 --- a/tox.ini +++ b/tox.ini @@ -91,7 +91,6 @@ addopts= # ignore=D000 [flake8] -# ignore= -# E124 -# E303 -# W503 +exclude=tests/test_djangocache.py +extend-ignore=E203 +max-line-length=120 From 99f0ca2439f6c98ded51d555658b7d538f09010c Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Fri, 22 Jan 2021 11:53:46 -0800 Subject: [PATCH 037/126] Flake8 fixes (mostly removing useless module imports) --- tests/benchmark_core.py | 9 ++++----- tests/benchmark_djangocache.py | 10 ++++------ tests/issue_85.py | 2 +- tests/plot.py | 2 +- tests/plot_early_recompute.py | 2 +- tests/settings_benchmark.py | 4 ++-- tests/stress_test_core.py | 7 +++---- tests/stress_test_deque.py | 1 - tests/stress_test_deque_mp.py | 1 - tests/stress_test_fanout.py | 4 +--- tests/test_core.py | 9 ++------- tests/test_deque.py | 5 ----- tests/test_doctest.py | 2 -- tests/test_fanout.py | 9 ++------- tests/test_index.py | 3 --- tests/test_recipes.py | 1 - tests/utils.py | 19 ------------------- 17 files changed, 21 insertions(+), 69 deletions(-) diff --git a/tests/benchmark_core.py b/tests/benchmark_core.py index a048b79..282ce2a 100644 --- a/tests/benchmark_core.py +++ b/tests/benchmark_core.py @@ -12,7 +12,6 @@ import pickle import random import shutil -import sys import time import warnings @@ -30,7 +29,7 @@ # Disk Cache Benchmarks ############################################################################### -import diskcache +import diskcache # noqa caches.append( ( @@ -123,13 +122,13 @@ def worker(num, kind, args, kwargs): start = time.time() result = obj.set(key, value) end = time.time() - miss = result == False + miss = result is False action = 'set' else: start = time.time() result = obj.delete(key) end = time.time() - miss = result == False + miss = result is False action = 'delete' if count > WARMUP: @@ -154,7 +153,7 @@ def dispatch(): try: obj.close() - except: + except Exception: pass processes = [ diff --git a/tests/benchmark_djangocache.py b/tests/benchmark_djangocache.py index 0de0424..9dbbcd7 100644 --- a/tests/benchmark_djangocache.py +++ b/tests/benchmark_djangocache.py @@ -12,9 +12,7 @@ import pickle import random import shutil -import sys import time -import warnings from utils import display @@ -59,13 +57,13 @@ def worker(num, name): start = time.time() result = obj.set(key, value) end = time.time() - miss = result == False + miss = result is False action = 'set' else: start = time.time() result = obj.delete(key) end = time.time() - miss = result == False + miss = result is False action = 'delete' if count > WARMUP: @@ -91,14 +89,14 @@ def prepare(name): try: obj.close() - except: + except Exception: pass def dispatch(): setup() - from django.core.cache import caches + from django.core.cache import caches # noqa for name in ['locmem', 'memcached', 'redis', 'diskcache', 'filebased']: shutil.rmtree('tmp', ignore_errors=True) diff --git a/tests/issue_85.py b/tests/issue_85.py index b325df0..723406b 100644 --- a/tests/issue_85.py +++ b/tests/issue_85.py @@ -109,7 +109,7 @@ def run(statements): shard._sql(statement) if index == 0: values.append(('BEGIN', ident)) - except sqlite3.OperationalError as exc: + except sqlite3.OperationalError: values.append(('ERROR', ident)) diff --git a/tests/plot.py b/tests/plot.py index 670bf49..2138659 100644 --- a/tests/plot.py +++ b/tests/plot.py @@ -48,7 +48,7 @@ def parse_data(infile): if blocks.match(line): try: name = title.match(lines[index + 1]).group(1) - except: + except Exception: index += 1 continue diff --git a/tests/plot_early_recompute.py b/tests/plot_early_recompute.py index 7096ea0..e58f580 100644 --- a/tests/plot_early_recompute.py +++ b/tests/plot_early_recompute.py @@ -22,7 +22,7 @@ def timer(func): @ft.wraps(func) def wrapper(*args, **kwargs): start = time.time() - result = func(*args, **kwargs) + func(*args, **kwargs) pair = start, time.time() with lock: times.append(pair) diff --git a/tests/settings_benchmark.py b/tests/settings_benchmark.py index 1724cef..c734e68 100644 --- a/tests/settings_benchmark.py +++ b/tests/settings_benchmark.py @@ -1,9 +1,9 @@ -from .settings import * +from .settings import * # noqa CACHES = { 'default': { 'BACKEND': 'diskcache.DjangoCache', - 'LOCATION': CACHE_DIR, + 'LOCATION': CACHE_DIR, # noqa }, 'memcached': { 'BACKEND': 'django.core.cache.backends.memcached.PyLibMCCache', diff --git a/tests/stress_test_core.py b/tests/stress_test_core.py index 6c48d5c..6fd0991 100644 --- a/tests/stress_test_core.py +++ b/tests/stress_test_core.py @@ -7,7 +7,6 @@ import queue import random import shutil -import sys import threading import time import warnings @@ -196,7 +195,7 @@ def dispatch(num, eviction_policy, processes, threads): for thread_queue in thread_queues: thread_queue.put(None) - start = time.time() + # start = time.time() for thread in subthreads: thread.start() @@ -204,7 +203,7 @@ def dispatch(num, eviction_policy, processes, threads): for thread in subthreads: thread.join() - stop = time.time() + # stop = time.time() timings = co.defaultdict(list) @@ -396,7 +395,7 @@ def stress_test_mp(): parser.add_argument( '-v', '--eviction-policy', - type=unicode, + type=str, default=u'least-recently-stored', ) diff --git a/tests/stress_test_deque.py b/tests/stress_test_deque.py index 7b3ac2f..845b2c2 100644 --- a/tests/stress_test_deque.py +++ b/tests/stress_test_deque.py @@ -2,7 +2,6 @@ import collections as co import functools as ft -import itertools as it import random import diskcache as dc diff --git a/tests/stress_test_deque_mp.py b/tests/stress_test_deque_mp.py index db7b5c4..091ff0e 100644 --- a/tests/stress_test_deque_mp.py +++ b/tests/stress_test_deque_mp.py @@ -1,6 +1,5 @@ """Stress test diskcache.persistent.Deque.""" -import functools as ft import itertools as it import multiprocessing as mp import os diff --git a/tests/stress_test_fanout.py b/tests/stress_test_fanout.py index 1f29987..58708c9 100644 --- a/tests/stress_test_fanout.py +++ b/tests/stress_test_fanout.py @@ -1,13 +1,11 @@ "Stress test diskcache.core.Cache." -import collections as co import multiprocessing as mp import os import pickle import queue import random import shutil -import sys import threading import time import warnings @@ -388,7 +386,7 @@ def stress_test_mp(): parser.add_argument( '-v', '--eviction-policy', - type=unicode, + type=str, default=u'least-recently-stored', ) diff --git a/tests/test_core.py b/tests/test_core.py index 6fe50c1..bcc79e1 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1,22 +1,17 @@ "Test diskcache.core.Cache." -import collections as co import errno -import functools as ft import hashlib import io import os import os.path as op import pickle -import random import shutil import sqlite3 import subprocess as sp -import sys import tempfile import threading import time -import unittest import warnings from unittest import mock @@ -130,7 +125,7 @@ def test_init_makedirs(): with pytest.raises(EnvironmentError): try: with mock.patch('os.makedirs', makedirs): - cache = dc.Cache(cache_dir) + dc.Cache(cache_dir) except EnvironmentError: shutil.rmtree(cache_dir, ignore_errors=True) raise @@ -244,7 +239,7 @@ def test_read(cache): def test_read_keyerror(cache): with pytest.raises(KeyError): - with cache.read(0) as reader: + with cache.read(0): pass diff --git a/tests/test_deque.py b/tests/test_deque.py index bfb9e7d..8113dfe 100644 --- a/tests/test_deque.py +++ b/tests/test_deque.py @@ -1,6 +1,5 @@ "Test diskcache.persistent.Deque." -import functools as ft import pickle import shutil import tempfile @@ -278,7 +277,3 @@ def test_rotate_indexerror_negative(deque): with mock.patch.object(deque, '_cache', cache): deque.rotate(-1) - - -def test_repr(deque): - assert repr(deque).startswith('Deque(') diff --git a/tests/test_doctest.py b/tests/test_doctest.py index 70fa61c..822d8db 100644 --- a/tests/test_doctest.py +++ b/tests/test_doctest.py @@ -1,6 +1,4 @@ import doctest -import shutil -import sys import diskcache.core import diskcache.djangocache diff --git a/tests/test_fanout.py b/tests/test_fanout.py index effe47d..8918af3 100644 --- a/tests/test_fanout.py +++ b/tests/test_fanout.py @@ -1,18 +1,13 @@ "Test diskcache.fanout.FanoutCache." import collections as co -import errno -import functools as ft import hashlib import io import os import os.path as op import pickle -import random import shutil -import sqlite3 import subprocess as sp -import sys import tempfile import threading import time @@ -67,7 +62,7 @@ def test_set_get_delete(cache): for value in range(100): assert cache.delete(value) - assert cache.delete(100) == False + assert cache.delete(100) is False cache.check() @@ -353,7 +348,7 @@ def test_read(cache): def test_read_keyerror(cache): with pytest.raises(KeyError): - with cache.read(0) as reader: + with cache.read(0): pass diff --git a/tests/test_index.py b/tests/test_index.py index 8852e99..27639f7 100644 --- a/tests/test_index.py +++ b/tests/test_index.py @@ -1,11 +1,8 @@ "Test diskcache.persistent.Index." -import functools as ft import pickle import shutil -import sys import tempfile -from unittest import mock import pytest diff --git a/tests/test_recipes.py b/tests/test_recipes.py index efaa47c..8d13f2b 100644 --- a/tests/test_recipes.py +++ b/tests/test_recipes.py @@ -3,7 +3,6 @@ import shutil import threading import time -from unittest import mock import pytest diff --git a/tests/utils.py b/tests/utils.py index 504b930..5b41ce9 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -18,7 +18,6 @@ def percentile(sequence, percent): def secs(value): units = ['s ', 'ms', 'us', 'ns'] - pos = 0 if value is None: return ' 0.000ns' @@ -60,24 +59,6 @@ def unmount_ramdisk(dev_path, path): run('rm', '-r', path) -def retry(sql, query): - pause = 0.001 - error = sqlite3.OperationalError - - for _ in range(int(LIMITS[u'timeout'] / pause)): - try: - sql(query).fetchone() - except sqlite3.OperationalError as exc: - error = exc - time.sleep(pause) - else: - break - else: - raise error - - del error - - def display(name, timings): cols = ('Action', 'Count', 'Miss', 'Median', 'P90', 'P99', 'Max', 'Total') template = ' '.join(['%9s'] * len(cols)) From 4b0fc2342d87f59ca1a024d3f74332dd53e3cb9e Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Fri, 22 Jan 2021 12:16:28 -0800 Subject: [PATCH 038/126] Update pylint and fix code --- .pylintrc | 90 ++++++++++++++++++++++++++------------------ diskcache/core.py | 6 +-- diskcache/fanout.py | 2 +- diskcache/recipes.py | 8 ++-- 4 files changed, 61 insertions(+), 45 deletions(-) diff --git a/.pylintrc b/.pylintrc index 8458ed4..158fe34 100644 --- a/.pylintrc +++ b/.pylintrc @@ -5,6 +5,9 @@ # run arbitrary code. extension-pkg-whitelist= +# Specify a score threshold to be exceeded before program exits with error. +fail-under=10.0 + # Add files or directories to the blacklist. They should be base names, not # paths. ignore=CVS @@ -26,16 +29,13 @@ jobs=1 # complex, nested conditions. limit-inference-results=100 -# List of plugins (as comma separated values of python modules names) to load, +# List of plugins (as comma separated values of python module names) to load, # usually to register additional checkers. load-plugins= # Pickle collected data for later comparisons. persistent=yes -# Specify a configuration file. -#rcfile= - # When enabled, pylint would attempt to guess common misconfiguration and emit # user-friendly hints instead of false-positive error messages. suggestion-mode=yes @@ -139,12 +139,10 @@ disable=print-statement, deprecated-sys-function, exception-escape, comprehension-escape, - no-else-return, no-member, - useless-object-inheritance, + no-else-return, + duplicate-code, inconsistent-return-statements, - ungrouped-imports, - not-callable, # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option @@ -155,11 +153,11 @@ enable=c-extension-no-member [REPORTS] -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables errors warning, statement which -# respectively contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (RP0004). +# Python expression which should return a score less than or equal to 10. You +# have access to the variables 'error', 'warning', 'refactor', and 'convention' +# which contain the number of messages in each category, as well as 'statement' +# which is the total number of statements analyzed. This score is used by the +# global evaluation report (RP0004). evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) # Template used to display messages. This is a python new-style format string @@ -181,7 +179,7 @@ score=yes [REFACTORING] # Maximum number of nested blocks for function / method body -max-nested-blocks=6 +max-nested-blocks=5 # Complete name of functions that never returns. When checking for # inconsistent-return-statements if a never returning function is called then @@ -192,8 +190,8 @@ never-returning-functions=sys.exit [LOGGING] -# Format style used to check logging format string. `old` means using % -# formatting, while `new` is for `{}` formatting. +# The type of string formatting that logging methods do. `old` means using % +# formatting, `new` is for `{}` formatting. logging-format-style=old # Logging modules to check that the string format arguments are in logging @@ -206,18 +204,18 @@ logging-modules=logging # Limits count of emitted suggestions for spelling mistakes. max-spelling-suggestions=4 -# Spelling dictionary name. Available dictionaries: none. To make it working -# install python-enchant package.. +# Spelling dictionary name. Available dictionaries: none. To make it work, +# install the python-enchant package. spelling-dict= # List of comma separated words that should not be checked. spelling-ignore-words= -# A path to a file that contains private dictionary; one word per line. +# A path to a file that contains the private dictionary; one word per line. spelling-private-dict-file= -# Tells whether to store unknown words to indicated private dictionary in -# --spelling-private-dict-file option instead of raising a message. +# Tells whether to store unknown words to the private dictionary (see the +# --spelling-private-dict-file option) instead of raising a message. spelling-store-unknown-words=no @@ -228,6 +226,9 @@ notes=FIXME, XXX, TODO +# Regular expression of note tags to take in consideration. +#notes-rgx= + [TYPECHECK] @@ -264,7 +265,7 @@ ignored-classes=optparse.Values,thread._local,_thread._local # List of module names for which member attributes should not be checked # (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis. It +# and thus existing member attributes cannot be deduced by static analysis). It # supports qualified module names, as well as Unix pattern matching. ignored-modules= @@ -280,6 +281,9 @@ missing-member-hint-distance=1 # showing a hint for a missing member. missing-member-max-choices=1 +# List of decorators that change the signature of a decorated function. +signature-mutators= + [VARIABLES] @@ -330,14 +334,7 @@ indent-string=' ' max-line-length=100 # Maximum number of lines in a module. -max-module-lines=2500 - -# List of optional constructs for which whitespace checking is disabled. `dict- -# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. -# `trailing-comma` allows a space between comma and closing bracket: (a, ). -# `empty-line` allows space-only lines. -no-space-check=trailing-comma, - dict-separator +max-module-lines=3000 # Allow the body of a class to be on the same line as the declaration if body # contains single statement. @@ -357,10 +354,10 @@ ignore-comments=yes ignore-docstrings=yes # Ignore imports when computing similarities. -ignore-imports=no +ignore-imports=yes # Minimum lines number of a similarity. -min-similarity-lines=9 +min-similarity-lines=4 [BASIC] @@ -387,6 +384,10 @@ bad-names=foo, tutu, tata +# Bad variable names regexes, separated by a comma. If names match any regex, +# they will always be refused +bad-names-rgxs= + # Naming style matching correct class attribute names. class-attribute-naming-style=any @@ -427,6 +428,10 @@ good-names=i, Run, _ +# Good variable names regexes, separated by a comma. If names match any regex, +# they will always be accepted +good-names-rgxs= + # Include a hint for the correct naming format with invalid-name. include-naming-hint=no @@ -474,14 +479,21 @@ variable-naming-style=snake_case [STRING] -# This flag controls whether the implicit-str-concat-in-sequence should -# generate a warning on implicit string concatenation in sequences defined over -# several lines. +# This flag controls whether inconsistent-quotes generates a warning when the +# character used as a quote delimiter is used inconsistently within a module. +check-quote-consistency=no + +# This flag controls whether the implicit-str-concat should generate a warning +# on implicit string concatenation in sequences defined over several lines. check-str-concat-over-line-jumps=no [IMPORTS] +# List of modules that can be imported at any level, not just the top level +# one. +allow-any-import-level= + # Allow wildcard imports from modules that define __all__. allow-wildcard-with-all=no @@ -512,13 +524,17 @@ known-standard-library= # Force import order to recognize a module as part of a third party library. known-third-party=enchant +# Couples of modules and preferred modules, separated by a comma. +preferred-modules= + [CLASSES] # List of method names used to declare (i.e. assign) instance attributes. defining-attr-methods=__init__, __new__, - setUp + setUp, + __post_init__ # List of member names, which should be excluded from the protected access # warning. @@ -543,7 +559,7 @@ max-args=8 # Maximum number of attributes for a class (see R0902). max-attributes=8 -# Maximum number of boolean expressions in an if statement. +# Maximum number of boolean expressions in an if statement (see R0916). max-bool-expr=5 # Maximum number of branch for function / method body. diff --git a/diskcache/core.py b/diskcache/core.py index 419c64f..da4d884 100644 --- a/diskcache/core.py +++ b/diskcache/core.py @@ -109,7 +109,7 @@ def __repr__(self): } -class Disk(object): +class Disk: "Cache key and value serialization for SQLite database and files." def __init__(self, directory, min_file_size=0, pickle_protocol=0): @@ -418,7 +418,7 @@ def args_to_key(base, args, kwargs, typed): return key -class Cache(object): +class Cache: "Disk and file backed cache." def __init__(self, directory=None, timeout=60, disk=Disk, **settings): @@ -2138,7 +2138,7 @@ def cull(self, retry=False): select_policy = EVICTION_POLICY[self.eviction_policy]['cull'] if select_policy is None: - return + return 0 select_filename = select_policy.format(fields='filename', now=now) diff --git a/diskcache/fanout.py b/diskcache/fanout.py index 6a13d6e..99384a0 100644 --- a/diskcache/fanout.py +++ b/diskcache/fanout.py @@ -13,7 +13,7 @@ from .persistent import Deque, Index -class FanoutCache(object): +class FanoutCache: "Cache that shards keys and values." def __init__( diff --git a/diskcache/recipes.py b/diskcache/recipes.py index 3acddd8..8f7bd32 100644 --- a/diskcache/recipes.py +++ b/diskcache/recipes.py @@ -12,7 +12,7 @@ from .core import ENOVAL, args_to_key, full_name -class Averager(object): +class Averager: """Recipe for calculating a running average. Sometimes known as "online statistics," the running average maintains the @@ -63,7 +63,7 @@ def pop(self): return None if count == 0 else total / count -class Lock(object): +class Lock: """Recipe for cross-process and cross-thread lock. >>> import diskcache @@ -111,7 +111,7 @@ def __exit__(self, *exc_info): self.release() -class RLock(object): +class RLock: """Recipe for cross-process and cross-thread re-entrant lock. >>> import diskcache @@ -179,7 +179,7 @@ def __exit__(self, *exc_info): self.release() -class BoundedSemaphore(object): +class BoundedSemaphore: """Recipe for cross-process and cross-thread bounded semaphore. >>> import diskcache From c654aeda6a969eeadc5fd81e1de5faab24a832a4 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Fri, 22 Jan 2021 12:31:18 -0800 Subject: [PATCH 039/126] Update Sphinx and re-gen conf.py --- docs/Makefile | 222 +++-------------------------------------- docs/conf.py | 271 +++++--------------------------------------------- docs/make.bat | 246 ++------------------------------------------- 3 files changed, 46 insertions(+), 693 deletions(-) diff --git a/docs/Makefile b/docs/Makefile index e3bd50b..d4bb2cb 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -1,216 +1,20 @@ -# Makefile for Sphinx documentation +# Minimal makefile for Sphinx documentation # -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -PAPER = +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . BUILDDIR = _build -# User-friendly check for sphinx-build -ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) -$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) -endif - -# Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -# the i18n builder cannot share the environment and doctrees with the others -I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . - -.PHONY: help +# Put it first so that "make" without argument is like "make help". help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " dirhtml to make HTML files named index.html in directories" - @echo " singlehtml to make a single large HTML file" - @echo " pickle to make pickle files" - @echo " json to make JSON files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make HTML files and a qthelp project" - @echo " applehelp to make an Apple Help Book" - @echo " devhelp to make HTML files and a Devhelp project" - @echo " epub to make an epub" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " latexpdf to make LaTeX files and run them through pdflatex" - @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" - @echo " text to make text files" - @echo " man to make manual pages" - @echo " texinfo to make Texinfo files" - @echo " info to make Texinfo files and run them through makeinfo" - @echo " gettext to make PO message catalogs" - @echo " changes to make an overview of all changed/added/deprecated items" - @echo " xml to make Docutils-native XML files" - @echo " pseudoxml to make pseudoxml-XML files for display purposes" - @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" - @echo " coverage to run coverage check of the documentation (if enabled)" - -.PHONY: clean -clean: - rm -rf $(BUILDDIR)/* - -.PHONY: html -html: - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -.PHONY: dirhtml -dirhtml: - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." - -.PHONY: singlehtml -singlehtml: - $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml - @echo - @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." - -.PHONY: pickle -pickle: - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle - @echo - @echo "Build finished; now you can process the pickle files." - -.PHONY: json -json: - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json - @echo - @echo "Build finished; now you can process the JSON files." - -.PHONY: htmlhelp -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." - -.PHONY: qthelp -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/DiskCache.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/DiskCache.qhc" - -.PHONY: applehelp -applehelp: - $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp - @echo - @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." - @echo "N.B. You won't be able to view it unless you put it in" \ - "~/Library/Documentation/Help or install it in your application" \ - "bundle." - -.PHONY: devhelp -devhelp: - $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp - @echo - @echo "Build finished." - @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/DiskCache" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/DiskCache" - @echo "# devhelp" - -.PHONY: epub -epub: - $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub - @echo - @echo "Build finished. The epub file is in $(BUILDDIR)/epub." - -.PHONY: latex -latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo - @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." - @echo "Run \`make' in that directory to run these through (pdf)latex" \ - "(use \`make latexpdf' here to do that automatically)." - -.PHONY: latexpdf -latexpdf: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through pdflatex..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -.PHONY: latexpdfja -latexpdfja: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through platex and dvipdfmx..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -.PHONY: text -text: - $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text - @echo - @echo "Build finished. The text files are in $(BUILDDIR)/text." - -.PHONY: man -man: - $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man - @echo - @echo "Build finished. The manual pages are in $(BUILDDIR)/man." - -.PHONY: texinfo -texinfo: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo - @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." - @echo "Run \`make' in that directory to run these through makeinfo" \ - "(use \`make info' here to do that automatically)." - -.PHONY: info -info: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo "Running Texinfo files through makeinfo..." - make -C $(BUILDDIR)/texinfo info - @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." - -.PHONY: gettext -gettext: - $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale - @echo - @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." - -.PHONY: changes -changes: - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes - @echo - @echo "The overview file is in $(BUILDDIR)/changes." - -.PHONY: linkcheck -linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck - @echo - @echo "Link check complete; look for any errors in the above output " \ - "or in $(BUILDDIR)/linkcheck/output.txt." - -.PHONY: doctest -doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest - @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." - -.PHONY: coverage -coverage: - $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage - @echo "Testing of coverage in the sources finished, look at the " \ - "results in $(BUILDDIR)/coverage/python.txt." + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -.PHONY: xml -xml: - $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml - @echo - @echo "Build finished. The XML files are in $(BUILDDIR)/xml." +.PHONY: help Makefile -.PHONY: pseudoxml -pseudoxml: - $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml - @echo - @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/conf.py b/docs/conf.py index 2556188..402d3ad 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,31 +1,33 @@ -# -*- coding: utf-8 -*- +# Configuration file for the Sphinx documentation builder. # -# DiskCache documentation build configuration file, created by -# sphinx-quickstart on Wed Feb 10 20:20:15 2016. -# -# This file is execfile()d with the current directory set to its -# containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html -import sys -import os +# -- Path setup -------------------------------------------------------------- # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. + +import os +import sys sys.path.insert(0, os.path.abspath('..')) + import diskcache -from diskcache import __version__ -# -- General configuration ------------------------------------------------ -# If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +# -- Project information ----------------------------------------------------- + +project = 'DiskCache' +copyright = '2021, Grant Jenks' +author = 'Grant Jenks' + +# The full version, including alpha/beta/rc tags +release = diskcache.__version__ + + +# -- General configuration --------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom @@ -38,77 +40,13 @@ # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] -# The suffix(es) of source filenames. -# You can specify multiple suffix as a list of string: -# source_suffix = ['.rst', '.md'] -source_suffix = '.rst' - -# The encoding of source files. -#source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = 'index' - -# General information about the project. -project = u'DiskCache' -copyright = u'2020, Grant Jenks' -author = u'Grant Jenks' - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = __version__ -# The full version, including alpha/beta/rc tags. -release = version - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# -# This is also used if you do content translation via gettext catalogs. -# Usually you set "language" from the command line for these cases. -language = None - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -#today = '' -# Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' - # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ['_build'] - -# The reST default role (used for this markup: `text`) to use for all -# documents. -#default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -#add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -#show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' - -# A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] - -# If true, keep warnings as "system message" paragraphs in the built documents. -#keep_warnings = False - -# If true, `todo` and `todoList` produce output, else they produce nothing. -todo_include_todos = True +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] -# -- Options for HTML output ---------------------------------------------- +# -- Options for HTML output ------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. @@ -130,43 +68,11 @@ 'github_type': 'star', } -# Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -#html_title = None - -# A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -#html_logo = None - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -#html_favicon = None - # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] -# Add any extra paths that contain custom files (such as robots.txt or -# .htaccess) here, relative to this directory. These files are copied -# directly to the root of the documentation. -#html_extra_path = [] - -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -#html_use_smartypants = True - # Custom sidebar templates, maps document names to template names. html_sidebars = { '**': [ @@ -178,134 +84,5 @@ ] } -# Additional templates that should be rendered to pages, maps page names to -# template names. -#html_additional_pages = {} - -# If false, no module index is generated. -#html_domain_indices = True - -# If false, no index is generated. -#html_use_index = True - -# If true, the index is split into individual pages for each letter. -#html_split_index = False - -# If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True - -# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True - -# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -#html_use_opensearch = '' - -# This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None - -# Language to be used for generating the HTML full-text search index. -# Sphinx supports the following languages: -# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' -# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' -#html_search_language = 'en' - -# A dictionary with options for the search language support, empty by default. -# Now only 'ja' uses this config value -#html_search_options = {'type': 'default'} - -# The name of a javascript file (relative to the configuration directory) that -# implements a search results scorer. If empty, the default will be used. -#html_search_scorer = 'scorer.js' - -# Output file base name for HTML help builder. -htmlhelp_basename = 'DiskCacheDoc' - def setup(app): - app.add_stylesheet('custom.css') - - -# -- Options for LaTeX output --------------------------------------------- - -latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', - -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', - -# Additional stuff for the LaTeX preamble. -#'preamble': '', - -# Latex figure (float) alignment -#'figure_align': 'htbp', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - (master_doc, 'DiskCache.tex', u'DiskCache Documentation', - u'Grant Jenks', 'manual'), -] - -# The name of an image file (relative to this directory) to place at the top of -# the title page. -#latex_logo = None - -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -#latex_use_parts = False - -# If true, show page references after internal links. -#latex_show_pagerefs = False - -# If true, show URL addresses after external links. -#latex_show_urls = False - -# Documents to append as an appendix to all manuals. -#latex_appendices = [] - -# If false, no module index is generated. -#latex_domain_indices = True - - -# -- Options for manual page output --------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'diskcache', u'DiskCache Documentation', - [author], 1) -] - -# If true, show URL addresses after external links. -#man_show_urls = False - - -# -- Options for Texinfo output ------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - (master_doc, 'DiskCache', u'DiskCache Documentation', - author, 'DiskCache', 'Disk and file backed cache.', - 'Miscellaneous'), -] - -# Documents to append as an appendix to all manuals. -#texinfo_appendices = [] - -# If false, no module index is generated. -#texinfo_domain_indices = True - -# How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' - -# If true, do not generate a @detailmenu in the "Top" node's menu. -#texinfo_no_detailmenu = False + app.add_css_file('custom.css') diff --git a/docs/make.bat b/docs/make.bat index e1a063b..2119f51 100644 --- a/docs/make.bat +++ b/docs/make.bat @@ -1,62 +1,18 @@ @ECHO OFF +pushd %~dp0 + REM Command file for Sphinx documentation if "%SPHINXBUILD%" == "" ( set SPHINXBUILD=sphinx-build ) +set SOURCEDIR=. set BUILDDIR=_build -set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . -set I18NSPHINXOPTS=%SPHINXOPTS% . -if NOT "%PAPER%" == "" ( - set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% - set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% -) if "%1" == "" goto help -if "%1" == "help" ( - :help - echo.Please use `make ^` where ^ is one of - echo. html to make standalone HTML files - echo. dirhtml to make HTML files named index.html in directories - echo. singlehtml to make a single large HTML file - echo. pickle to make pickle files - echo. json to make JSON files - echo. htmlhelp to make HTML files and a HTML help project - echo. qthelp to make HTML files and a qthelp project - echo. devhelp to make HTML files and a Devhelp project - echo. epub to make an epub - echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter - echo. text to make text files - echo. man to make manual pages - echo. texinfo to make Texinfo files - echo. gettext to make PO message catalogs - echo. changes to make an overview over all changed/added/deprecated items - echo. xml to make Docutils-native XML files - echo. pseudoxml to make pseudoxml-XML files for display purposes - echo. linkcheck to check all external links for integrity - echo. doctest to run all doctests embedded in the documentation if enabled - echo. coverage to run coverage check of the documentation if enabled - goto end -) - -if "%1" == "clean" ( - for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i - del /q /s %BUILDDIR%\* - goto end -) - - -REM Check if sphinx-build is available and fallback to Python version if any -%SPHINXBUILD% 1>NUL 2>NUL -if errorlevel 9009 goto sphinx_python -goto sphinx_ok - -:sphinx_python - -set SPHINXBUILD=python -m sphinx.__init__ -%SPHINXBUILD% 2> nul +%SPHINXBUILD% >NUL 2>NUL if errorlevel 9009 ( echo. echo.The 'sphinx-build' command was not found. Make sure you have Sphinx @@ -69,195 +25,11 @@ if errorlevel 9009 ( exit /b 1 ) -:sphinx_ok - - -if "%1" == "html" ( - %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/html. - goto end -) - -if "%1" == "dirhtml" ( - %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. - goto end -) - -if "%1" == "singlehtml" ( - %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. - goto end -) - -if "%1" == "pickle" ( - %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can process the pickle files. - goto end -) - -if "%1" == "json" ( - %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can process the JSON files. - goto end -) - -if "%1" == "htmlhelp" ( - %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run HTML Help Workshop with the ^ -.hhp project file in %BUILDDIR%/htmlhelp. - goto end -) +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end -if "%1" == "qthelp" ( - %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run "qcollectiongenerator" with the ^ -.qhcp project file in %BUILDDIR%/qthelp, like this: - echo.^> qcollectiongenerator %BUILDDIR%\qthelp\DiskCache.qhcp - echo.To view the help file: - echo.^> assistant -collectionFile %BUILDDIR%\qthelp\DiskCache.ghc - goto end -) - -if "%1" == "devhelp" ( - %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. - goto end -) - -if "%1" == "epub" ( - %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The epub file is in %BUILDDIR%/epub. - goto end -) - -if "%1" == "latex" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "latexpdf" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - cd %BUILDDIR%/latex - make all-pdf - cd %~dp0 - echo. - echo.Build finished; the PDF files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "latexpdfja" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - cd %BUILDDIR%/latex - make all-pdf-ja - cd %~dp0 - echo. - echo.Build finished; the PDF files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "text" ( - %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The text files are in %BUILDDIR%/text. - goto end -) - -if "%1" == "man" ( - %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The manual pages are in %BUILDDIR%/man. - goto end -) - -if "%1" == "texinfo" ( - %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. - goto end -) - -if "%1" == "gettext" ( - %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The message catalogs are in %BUILDDIR%/locale. - goto end -) - -if "%1" == "changes" ( - %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes - if errorlevel 1 exit /b 1 - echo. - echo.The overview file is in %BUILDDIR%/changes. - goto end -) - -if "%1" == "linkcheck" ( - %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck - if errorlevel 1 exit /b 1 - echo. - echo.Link check complete; look for any errors in the above output ^ -or in %BUILDDIR%/linkcheck/output.txt. - goto end -) - -if "%1" == "doctest" ( - %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest - if errorlevel 1 exit /b 1 - echo. - echo.Testing of doctests in the sources finished, look at the ^ -results in %BUILDDIR%/doctest/output.txt. - goto end -) - -if "%1" == "coverage" ( - %SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage - if errorlevel 1 exit /b 1 - echo. - echo.Testing of coverage in the sources finished, look at the ^ -results in %BUILDDIR%/coverage/python.txt. - goto end -) - -if "%1" == "xml" ( - %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The XML files are in %BUILDDIR%/xml. - goto end -) - -if "%1" == "pseudoxml" ( - %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. - goto end -) +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% :end +popd From d1cf8c528ae21fc8d2dff7afe91299df7e5cd1f7 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Fri, 22 Jan 2021 12:32:45 -0800 Subject: [PATCH 040/126] Update copyright year --- LICENSE | 2 +- README.rst | 4 ++-- diskcache/__init__.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/LICENSE b/LICENSE index d31985f..ca80a22 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright 2016-2020 Grant Jenks +Copyright 2016-2021 Grant Jenks Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the diff --git a/README.rst b/README.rst index ba2bf3e..969ea10 100644 --- a/README.rst +++ b/README.rst @@ -4,7 +4,7 @@ DiskCache: Disk Backed Cache `DiskCache`_ is an Apache2 licensed disk and file backed cache library, written in pure-Python, and compatible with Django. -The cloud-based computing of 2020 puts a premium on memory. Gigabytes of empty +The cloud-based computing of 2021 puts a premium on memory. Gigabytes of empty space is left on disks as processes vie for memory. Among these processes is Memcached (and sometimes Redis) which is used as a cache. Wouldn't it be nice to leverage empty disk space for caching? @@ -387,7 +387,7 @@ Reference License ------- -Copyright 2016-2020 Grant Jenks +Copyright 2016-2021 Grant Jenks Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the diff --git a/diskcache/__init__.py b/diskcache/__init__.py index befed76..c050a17 100644 --- a/diskcache/__init__.py +++ b/diskcache/__init__.py @@ -66,4 +66,4 @@ __build__ = 0x050100 __author__ = 'Grant Jenks' __license__ = 'Apache 2.0' -__copyright__ = 'Copyright 2016-2020 Grant Jenks' +__copyright__ = 'Copyright 2016-2021 Grant Jenks' From 6df650543aa3834af84346aefd4be85b74d993af Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Fri, 22 Jan 2021 12:34:12 -0800 Subject: [PATCH 041/126] Update readme badges and CI notes --- README.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.rst b/README.rst index 969ea10..b7e1ee2 100644 --- a/README.rst +++ b/README.rst @@ -77,16 +77,16 @@ Features - Thread-safe and process-safe - Supports multiple eviction policies (LRU and LFU included) - Keys support "tag" metadata and eviction -- Developed on Python 3.8 -- Tested on CPython 3.5, 3.6, 3.7, 3.8 +- Developed on Python 3.9 +- Tested on CPython 3.6, 3.7, 3.8, 3.9 - Tested on Linux, Mac OS X, and Windows -- Tested using Travis CI and AppVeyor CI +- Tested using GitHub Actions -.. image:: https://api.travis-ci.org/grantjenks/python-diskcache.svg?branch=master - :target: http://www.grantjenks.com/docs/diskcache/ +.. image:: https://github.com/grantjenks/python-diskcache/workflows/integration/badge.svg + :target: https://github.com/grantjenks/python-diskcache/actions?query=workflow%3Aintegration -.. image:: https://ci.appveyor.com/api/projects/status/github/grantjenks/python-diskcache?branch=master&svg=true - :target: http://www.grantjenks.com/docs/diskcache/ +.. image:: https://github.com/grantjenks/python-diskcache/workflows/release/badge.svg + :target: https://github.com/grantjenks/python-diskcache/actions?query=workflow%3Arelease Quickstart ---------- From 72127020640a383149b643933e89c7ebc577cc5f Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Fri, 22 Jan 2021 13:11:21 -0800 Subject: [PATCH 042/126] Pin jedi for ipython --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index a9023a1..2ab91c7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,6 +6,7 @@ django_redis doc8 flake8 ipython +jedi==0.17.* # Remove after IPython bug fixed. pickleDB pylibmc pylint From e49358a20ee359614a39d94047b27b9cfaf4ab1a Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Fri, 22 Jan 2021 13:33:58 -0800 Subject: [PATCH 043/126] Skip help() examples when running doctest --- README.rst | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/README.rst b/README.rst index b7e1ee2..acf6803 100644 --- a/README.rst +++ b/README.rst @@ -99,7 +99,7 @@ You can access documentation in the interpreter with Python's built-in help function:: >>> import diskcache - >>> help(diskcache) + >>> help(diskcache) # doctest: +SKIP The core of `DiskCache`_ is three data types intended for caching. `Cache`_ objects manage a SQLite database and filesystem directory to store key and @@ -107,26 +107,26 @@ value pairs. `FanoutCache`_ provides a sharding layer to utilize multiple caches and `DjangoCache`_ integrates that with `Django`_:: >>> from diskcache import Cache, FanoutCache, DjangoCache - >>> help(Cache) - >>> help(FanoutCache) - >>> help(DjangoCache) + >>> help(Cache) # doctest: +SKIP + >>> help(FanoutCache) # doctest: +SKIP + >>> help(DjangoCache) # doctest: +SKIP Built atop the caching data types, are `Deque`_ and `Index`_ which work as a cross-process, persistent replacements for Python's ``collections.deque`` and ``dict``. These implement the sequence and mapping container base classes:: >>> from diskcache import Deque, Index - >>> help(Deque) - >>> help(Index) + >>> help(Deque) # doctest: +SKIP + >>> help(Index) # doctest: +SKIP Finally, a number of `recipes`_ for cross-process synchronization are provided using an underlying cache. Features like memoization with cache stampede prevention, cross-process locking, and cross-process throttling are available:: >>> from diskcache import memoize_stampede, Lock, throttle - >>> help(memoize_stampede) - >>> help(Lock) - >>> help(throttle) + >>> help(memoize_stampede) # doctest: +SKIP + >>> help(Lock) # doctest: +SKIP + >>> help(throttle) # doctest: +SKIP Python's docstrings are a quick way to get started but not intended as a replacement for the `DiskCache Tutorial`_ and `DiskCache API Reference`_. From 0d41722a030695f4d988ac21383513c52f2aa073 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Fri, 22 Jan 2021 13:34:44 -0800 Subject: [PATCH 044/126] Fix configs for pytest --- tox.ini | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tox.ini b/tox.ini index d5f43c0..acf7b22 100644 --- a/tox.ini +++ b/tox.ini @@ -7,10 +7,12 @@ commands=pytest deps= django==2.2.* pytest + pytest-cov pytest-django pytest-xdist setenv= DJANGO_SETTINGS_MODULE=tests.settings + PYTHONPATH={toxinidir} [testenv:blue] commands=blue {toxinidir}/setup.py {toxinidir}/diskcache {toxinidir}/tests @@ -81,6 +83,8 @@ addopts= --cov-report=term-missing --cov=diskcache --doctest-glob="*.rst" + --ignore docs/case-study-web-crawler.rst + --ignore docs/sf-python-2017-meetup-talk.rst --ignore tests/benchmark_core.py --ignore tests/benchmark_djangocache.py --ignore tests/benchmark_glob.py From a5d1c73e38fbc917c5503e2e8bcd9a2f834efc3f Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Fri, 22 Jan 2021 13:39:33 -0800 Subject: [PATCH 045/126] Add branch coverage and decrease coverage minimum to 96 --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index acf7b22..04b7382 100644 --- a/tox.ini +++ b/tox.ini @@ -79,7 +79,7 @@ line_length = 79 addopts= -n auto --cov-branch - --cov-fail-under=100 + --cov-fail-under=96 --cov-report=term-missing --cov=diskcache --doctest-glob="*.rst" From fc38bd97009bd9ce352e8bf01950a2a22b5604b5 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Fri, 22 Jan 2021 13:43:31 -0800 Subject: [PATCH 046/126] Ignore more .coverage files --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index c496721..67157b8 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,7 @@ # test files/directories /.cache/ -.coverage +.coverage* .pytest_cache/ /.tox/ From 6b6944750acdaf80c6ce28eb8e7a35320ea3a3c0 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Fri, 22 Jan 2021 13:57:25 -0800 Subject: [PATCH 047/126] Bump version to 5.2.0 --- diskcache/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/diskcache/__init__.py b/diskcache/__init__.py index c050a17..d1021cd 100644 --- a/diskcache/__init__.py +++ b/diskcache/__init__.py @@ -62,8 +62,8 @@ pass __title__ = 'diskcache' -__version__ = '5.1.0' -__build__ = 0x050100 +__version__ = '5.2.0' +__build__ = 0x050200 __author__ = 'Grant Jenks' __license__ = 'Apache 2.0' __copyright__ = 'Copyright 2016-2021 Grant Jenks' From 619eb6904810440ef952ea425421315a7353ba12 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Fri, 22 Jan 2021 18:46:28 -0800 Subject: [PATCH 048/126] Install libmemcached-dev for release action --- .github/workflows/release.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 57c68e5..2a07787 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,6 +13,11 @@ jobs: steps: - uses: actions/checkout@v2 + - name: Install libmemcached-dev + run: | + sudo apt-get update + sudo apt-get install libmemcached-dev + - name: Set up Python uses: actions/setup-python@v2 with: From e1d7c4aaa6729178ca3216f4c8a75b835f963022 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Fri, 22 Jan 2021 18:47:55 -0800 Subject: [PATCH 049/126] Bump version to 5.2.1 --- diskcache/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/diskcache/__init__.py b/diskcache/__init__.py index d1021cd..428eb30 100644 --- a/diskcache/__init__.py +++ b/diskcache/__init__.py @@ -62,8 +62,8 @@ pass __title__ = 'diskcache' -__version__ = '5.2.0' -__build__ = 0x050200 +__version__ = '5.2.1' +__build__ = 0x050201 __author__ = 'Grant Jenks' __license__ = 'Apache 2.0' __copyright__ = 'Copyright 2016-2021 Grant Jenks' From 12c1db268d03699de86cf114facd2a5091bfc257 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Thu, 28 Jan 2021 11:11:33 +0200 Subject: [PATCH 050/126] Add Python 3.9 support trove classifier. --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 5f3cd88..7f0561c 100644 --- a/setup.py +++ b/setup.py @@ -47,6 +47,7 @@ def run_tests(self): 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: Implementation :: CPython', ), ) From 660e53d3afe5789babcf461e4cf7764e90dcf15a Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Sat, 30 Jan 2021 12:20:10 -0800 Subject: [PATCH 051/126] Run integration on pull requests --- .github/workflows/integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 7dfc198..0a44f89 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -1,6 +1,6 @@ name: integration -on: [push] +on: [push, pull_request] jobs: From 2ba659ece1cb7a2b99430cc805c5631f79ec9e8a Mon Sep 17 00:00:00 2001 From: Joakim Nordling Date: Sun, 28 Mar 2021 06:59:44 +0300 Subject: [PATCH 052/126] Fix typo --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index acf6803..c4aa8e9 100644 --- a/README.rst +++ b/README.rst @@ -185,7 +185,7 @@ other projects are shown in the tables below. access. Keys are arbitrary strings, values arbitrary pickle-able objects. * `pickleDB`_ is a lightweight and simple key-value store. It is built upon Python's simplejson module and was inspired by Redis. It is licensed with the - BSD three-caluse license. + BSD three-clause license. .. _`dbm`: https://docs.python.org/3/library/dbm.html .. _`shelve`: https://docs.python.org/3/library/shelve.html From 12400ec12eb32be37c41a4940b545797aae036a6 Mon Sep 17 00:00:00 2001 From: Abhilash Raj Date: Fri, 21 May 2021 14:44:18 -0700 Subject: [PATCH 053/126] Fix the URL to Django documentation for cache. --- docs/tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 8f8dc0f..58220b2 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -535,7 +535,7 @@ they are guaranteed to be stored in files. The full path is available on the file handle in the `name` attribute. Remember to also include the `Content-Type` header if known. -.. _`Django documentation on caching`: https://docs.djangoproject.com/en/1.9/topics/cache/#the-low-level-cache-api +.. _`Django documentation on caching`: https://docs.djangoproject.com/en/3.2/topics/cache/#the-low-level-cache-api Deque ----- From b460e7409d8d299e208500d640d69e55c1faccec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Gmach?= Date: Mon, 7 Jun 2021 15:11:34 +0200 Subject: [PATCH 054/126] remove leftovers from Travis and AppVeyor Both were removed in favor of GitHub actions. --- docs/conf.py | 1 - tests/stress_test_deque_mp.py | 6 ------ tests/stress_test_index_mp.py | 6 ------ 3 files changed, 13 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 402d3ad..d725198 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -59,7 +59,6 @@ 'logo': 'gj-logo.png', 'logo_name': True, 'logo_text_align': 'center', - 'travis_button': True, 'analytics_id': 'UA-19364636-2', 'show_powered_by': False, 'show_related': True, diff --git a/tests/stress_test_deque_mp.py b/tests/stress_test_deque_mp.py index 091ff0e..d6bb56a 100644 --- a/tests/stress_test_deque_mp.py +++ b/tests/stress_test_deque_mp.py @@ -107,12 +107,6 @@ def stress(seed, deque): def test(status=False): - if os.environ.get('TRAVIS') == 'true': - return - - if os.environ.get('APPVEYOR') == 'True': - return - random.seed(SEED) deque = dc.Deque(range(SIZE)) processes = [] diff --git a/tests/stress_test_index_mp.py b/tests/stress_test_index_mp.py index f8718f0..2d290ec 100644 --- a/tests/stress_test_index_mp.py +++ b/tests/stress_test_index_mp.py @@ -94,12 +94,6 @@ def stress(seed, index): def test(status=False): - if os.environ.get('TRAVIS') == 'true': - return - - if os.environ.get('APPVEYOR') == 'True': - return - random.seed(SEED) index = dc.Index(enumerate(range(KEYS))) processes = [] From d20ddb3b2273dee41e73aa72ba182f4331ccba83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Gmach?= Date: Tue, 8 Jun 2021 11:05:03 +0200 Subject: [PATCH 055/126] remove unused imports --- tests/stress_test_deque_mp.py | 1 - tests/stress_test_index_mp.py | 1 - 2 files changed, 2 deletions(-) diff --git a/tests/stress_test_deque_mp.py b/tests/stress_test_deque_mp.py index d6bb56a..f3b8a48 100644 --- a/tests/stress_test_deque_mp.py +++ b/tests/stress_test_deque_mp.py @@ -2,7 +2,6 @@ import itertools as it import multiprocessing as mp -import os import random import time diff --git a/tests/stress_test_index_mp.py b/tests/stress_test_index_mp.py index 2d290ec..06ed102 100644 --- a/tests/stress_test_index_mp.py +++ b/tests/stress_test_index_mp.py @@ -2,7 +2,6 @@ import itertools as it import multiprocessing as mp -import os import random import time From d9b9d06614a41beff9ebce9b1bc8038d5540394e Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Tue, 8 Jun 2021 21:42:54 -0700 Subject: [PATCH 056/126] Ignore pylint's consider-using-with in Disk.fetch --- diskcache/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/diskcache/core.py b/diskcache/core.py index da4d884..4d0ae05 100644 --- a/diskcache/core.py +++ b/diskcache/core.py @@ -261,7 +261,7 @@ def fetch(self, mode, filename, value, read): :return: corresponding Python value """ - # pylint: disable=no-self-use,unidiomatic-typecheck + # pylint: disable=no-self-use,unidiomatic-typecheck,consider-using-with if mode == MODE_RAW: return bytes(value) if type(value) is sqlite3.Binary else value elif mode == MODE_BINARY: From fac9ad3ffdc289336b7c280e50ca26712f65f8ea Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 30 Aug 2021 22:07:25 -0700 Subject: [PATCH 057/126] Simplify ENOENT handling around fetch() and remove() --- diskcache/core.py | 53 ++++++++++++++--------------------------------- 1 file changed, 15 insertions(+), 38 deletions(-) diff --git a/diskcache/core.py b/diskcache/core.py index 4d0ae05..1c915b1 100644 --- a/diskcache/core.py +++ b/diskcache/core.py @@ -26,14 +26,6 @@ def full_name(func): return func.__module__ + '.' + func.__qualname__ -try: - WindowsError -except NameError: - - class WindowsError(Exception): - "Windows error place-holder on platforms without support." - - class Constant(tuple): "Pretty display of immutable constant." @@ -328,13 +320,10 @@ def remove(self, filename): try: os.remove(full_path) - except WindowsError: + except OSError: + # OSError may occur if two caches attempt to delete the same + # file at the same time. pass - except OSError as error: - if error.errno != errno.ENOENT: - # ENOENT may occur if two caches attempt to delete the same - # file at the same time. - raise class JSONDisk(Disk): @@ -1201,13 +1190,10 @@ def get( try: value = self._disk.fetch(mode, filename, db_value, read) except IOError as error: - if error.errno == errno.ENOENT: - # Key was deleted before we could retrieve result. - if self.statistics: - sql(cache_miss) - return default - else: - raise + # Key was deleted before we could retrieve result. + if self.statistics: + sql(cache_miss) + return default if self.statistics: sql(cache_hit) @@ -1324,11 +1310,8 @@ def pop( try: value = self._disk.fetch(mode, filename, db_value, False) except IOError as error: - if error.errno == errno.ENOENT: - # Key was deleted before we could retrieve result. - return default - else: - raise + # Key was deleted before we could retrieve result. + return default finally: if filename is not None: self._disk.remove(filename) @@ -1595,10 +1578,8 @@ def pull( try: value = self._disk.fetch(mode, name, db_value, False) except IOError as error: - if error.errno == errno.ENOENT: - # Key was deleted before we could retrieve result. - continue - raise + # Key was deleted before we could retrieve result. + continue finally: if name is not None: self._disk.remove(name) @@ -1711,10 +1692,8 @@ def peek( try: value = self._disk.fetch(mode, name, db_value, False) except IOError as error: - if error.errno == errno.ENOENT: - # Key was deleted before we could retrieve result. - continue - raise + # Key was deleted before we could retrieve result. + continue finally: if name is not None: self._disk.remove(name) @@ -1794,10 +1773,8 @@ def peekitem(self, last=True, expire_time=False, tag=False, retry=False): try: value = self._disk.fetch(mode, name, db_value, False) except IOError as error: - if error.errno == errno.ENOENT: - # Key was deleted before we could retrieve result. - continue - raise + # Key was deleted before we could retrieve result. + continue break if expire_time and tag: From ceff81cde5ecacaf7ff79b50bd90250e988a5db0 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 30 Aug 2021 22:11:33 -0700 Subject: [PATCH 058/126] Add doc about IOError --- diskcache/core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/diskcache/core.py b/diskcache/core.py index 1c915b1..efa175b 100644 --- a/diskcache/core.py +++ b/diskcache/core.py @@ -251,6 +251,7 @@ def fetch(self, mode, filename, value, read): :param value: database value :param bool read: when True, return an open file handle :return: corresponding Python value + :raises: IOError if the value cannot be read """ # pylint: disable=no-self-use,unidiomatic-typecheck,consider-using-with From aefb2feda735b1f602eee290cac3a1baa95c8c8c Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 30 Aug 2021 22:12:36 -0700 Subject: [PATCH 059/126] Add notes about changes to store() and remove() --- diskcache/core.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/diskcache/core.py b/diskcache/core.py index efa175b..2cbcdad 100644 --- a/diskcache/core.py +++ b/diskcache/core.py @@ -187,6 +187,7 @@ def store(self, value, read, key=UNKNOWN): :return: (size, mode, filename, value) tuple for Cache table """ + # TODO: Retry mkdirs!!! # pylint: disable=unidiomatic-typecheck type_value = type(value) min_file_size = self.min_file_size @@ -317,6 +318,7 @@ def remove(self, filename): :param str filename: relative path to file """ + # TODO: Delete dir if empty!!! full_path = op.join(self._directory, filename) try: From 49c5979190bd33b134ab8acfe5b5c95823d567e2 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Tue, 31 Aug 2021 22:33:14 -0700 Subject: [PATCH 060/126] Update remove to cleanup parent dirs --- diskcache/core.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/diskcache/core.py b/diskcache/core.py index 2cbcdad..13a1b01 100644 --- a/diskcache/core.py +++ b/diskcache/core.py @@ -309,24 +309,26 @@ def filename(self, key=UNKNOWN, value=UNKNOWN): full_path = op.join(self._directory, filename) return filename, full_path - def remove(self, filename): - """Remove a file given by `filename`. + def remove(self, file_path): + """Remove a file given by `file_path`. - This method is cross-thread and cross-process safe. If an "error no - entry" occurs, it is suppressed. + This method is cross-thread and cross-process safe. If an OSError + occurs, it is suppressed. - :param str filename: relative path to file + :param str file_path: relative path to file """ - # TODO: Delete dir if empty!!! - full_path = op.join(self._directory, filename) + full_path = op.join(self._directory, file_path) + full_dir, _ = op.split(full_path) - try: + # Suppress OSError that may occur if two caches attempt to delete the + # same file or directory at the same time. + + with cl.suppress(OSError): os.remove(full_path) - except OSError: - # OSError may occur if two caches attempt to delete the same - # file at the same time. - pass + + with cl.suppress(OSError): + os.removedirs(full_dir) class JSONDisk(Disk): From 1acab3b7b0809180b7c5b0863751e18662662cc3 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Tue, 31 Aug 2021 22:33:38 -0700 Subject: [PATCH 061/126] Remove logic from filename() for creating directories --- diskcache/core.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/diskcache/core.py b/diskcache/core.py index 13a1b01..ac266e3 100644 --- a/diskcache/core.py +++ b/diskcache/core.py @@ -297,14 +297,6 @@ def filename(self, key=UNKNOWN, value=UNKNOWN): hex_name = codecs.encode(os.urandom(16), 'hex').decode('utf-8') sub_dir = op.join(hex_name[:2], hex_name[2:4]) name = hex_name[4:] + '.val' - directory = op.join(self._directory, sub_dir) - - try: - os.makedirs(directory) - except OSError as error: - if error.errno != errno.EEXIST: - raise - filename = op.join(sub_dir, name) full_path = op.join(self._directory, filename) return filename, full_path From b86aa9e0ba840821f2b5bcbc064a1283fd4d3e1e Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Tue, 31 Aug 2021 22:34:03 -0700 Subject: [PATCH 062/126] Modify store() to create the subdirs when writing the file (1 of 4) --- diskcache/core.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/diskcache/core.py b/diskcache/core.py index ac266e3..9040fb4 100644 --- a/diskcache/core.py +++ b/diskcache/core.py @@ -206,9 +206,26 @@ def store(self, value, read, key=UNKNOWN): return 0, MODE_RAW, None, sqlite3.Binary(value) else: filename, full_path = self.filename(key, value) + full_dir, _ = op.split(full_path) - with open(full_path, 'xb') as writer: - writer.write(value) + for count in range(11): + with cl.suppress(OSError): + os.makedirs(full_dir) + + try: + # Another cache may have deleted the directory before + # the file could be opened. + writer = open(full_path, 'xb') + except OSError: + if count == 10: + # Give up after 10 tries to open the file. + raise + continue + + with writer: + writer.write(value) + + break return len(value), MODE_BINARY, filename, None elif type_value is str: From 879a65a1932377d017cc187661fd880346459b0d Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 6 Sep 2021 22:02:21 -0700 Subject: [PATCH 063/126] Refactor file writing logic to retry makedirs --- diskcache/core.py | 66 +++++++++++++++++++++-------------------------- 1 file changed, 30 insertions(+), 36 deletions(-) diff --git a/diskcache/core.py b/diskcache/core.py index 9040fb4..332276a 100644 --- a/diskcache/core.py +++ b/diskcache/core.py @@ -187,7 +187,6 @@ def store(self, value, read, key=UNKNOWN): :return: (size, mode, filename, value) tuple for Cache table """ - # TODO: Retry mkdirs!!! # pylint: disable=unidiomatic-typecheck type_value = type(value) min_file_size = self.min_file_size @@ -206,46 +205,18 @@ def store(self, value, read, key=UNKNOWN): return 0, MODE_RAW, None, sqlite3.Binary(value) else: filename, full_path = self.filename(key, value) - full_dir, _ = op.split(full_path) - - for count in range(11): - with cl.suppress(OSError): - os.makedirs(full_dir) - - try: - # Another cache may have deleted the directory before - # the file could be opened. - writer = open(full_path, 'xb') - except OSError: - if count == 10: - # Give up after 10 tries to open the file. - raise - continue - - with writer: - writer.write(value) - - break - + self._write(full_path, io.BytesIO(value), 'xb') return len(value), MODE_BINARY, filename, None elif type_value is str: filename, full_path = self.filename(key, value) - - with open(full_path, 'x', encoding='UTF-8') as writer: - writer.write(value) - + self._write(full_path, io.StringIO(value), 'x', 'UTF-8') size = op.getsize(full_path) return size, MODE_TEXT, filename, None elif read: - size = 0 reader = ft.partial(value.read, 2 ** 22) filename, full_path = self.filename(key, value) - - with open(full_path, 'xb') as writer: - for chunk in iter(reader, b''): - size += len(chunk) - writer.write(chunk) - + iterator = iter(reader, b'') + size = self._write(full_path, iterator, 'xb') return size, MODE_BINARY, filename, None else: result = pickle.dumps(value, protocol=self.pickle_protocol) @@ -254,11 +225,34 @@ def store(self, value, read, key=UNKNOWN): return 0, MODE_PICKLE, None, sqlite3.Binary(result) else: filename, full_path = self.filename(key, value) + self._write(full_path, io.BytesIO(result), 'xb') + return len(result), MODE_PICKLE, filename, None + + def _write(self, full_path, iterator, mode, encoding=None): + full_dir, _ = op.split(full_path) - with open(full_path, 'xb') as writer: - writer.write(result) + for count in range(1, 11): + with cl.suppress(OSError): + os.makedirs(full_dir) - return len(result), MODE_PICKLE, filename, None + try: + # Another cache may have deleted the directory before + # the file could be opened. + writer = open(full_path, mode, encoding=encoding) + except OSError: + if count == 10: + # Give up after 10 tries to open the file. + raise + continue + + with writer: + size = 0 + for chunk in iterator: + size += len(chunk) + writer.write(chunk) + return size + + break def fetch(self, mode, filename, value, read): """Convert fields `mode`, `filename`, and `value` from Cache table to From 226a5cfd2ec68a726a1f6d535d6bc5c055cf226f Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 6 Sep 2021 22:39:29 -0700 Subject: [PATCH 064/126] Add test for Lock.locked() --- tests/test_recipes.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/test_recipes.py b/tests/test_recipes.py index 8d13f2b..88612cf 100644 --- a/tests/test_recipes.py +++ b/tests/test_recipes.py @@ -28,6 +28,26 @@ def test_averager(cache): assert nums.pop() == 9.5 +def test_lock(cache): + state = {'num': 0} + lock = dc.Lock(cache, 'demo') + + def worker(): + state['num'] += 1 + with lock: + assert lock.locked() + state['num'] += 1 + time.sleep(0.1) + + with lock: + thread = threading.Thread(target=worker) + thread.start() + time.sleep(0.1) + assert state['num'] == 1 + thread.join() + assert state['num'] == 2 + + def test_rlock(cache): state = {'num': 0} rlock = dc.RLock(cache, 'demo') From 9d5c0b7ebf43bea02eae968f75a9104a699fba46 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 6 Sep 2021 22:39:47 -0700 Subject: [PATCH 065/126] Test re-entrancy of "rlock" --- tests/test_recipes.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_recipes.py b/tests/test_recipes.py index 88612cf..3f330a6 100644 --- a/tests/test_recipes.py +++ b/tests/test_recipes.py @@ -55,8 +55,9 @@ def test_rlock(cache): def worker(): state['num'] += 1 with rlock: - state['num'] += 1 - time.sleep(0.1) + with rlock: + state['num'] += 1 + time.sleep(0.1) with rlock: thread = threading.Thread(target=worker) From 14094b31006d2951b64a8b87f2a07bd57efa6525 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 6 Sep 2021 22:40:23 -0700 Subject: [PATCH 066/126] Delete EACCESS error tests --- tests/test_core.py | 121 --------------------------------------------- 1 file changed, 121 deletions(-) diff --git a/tests/test_core.py b/tests/test_core.py index bcc79e1..a113443 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -333,26 +333,6 @@ def test_get_expired_slow_path(cache): assert cache.get(0) is None -def test_get_ioerror_slow_path(cache): - cache.reset('eviction_policy', 'least-recently-used') - cache.set(0, 0) - - disk = mock.Mock() - put = mock.Mock() - fetch = mock.Mock() - - disk.put = put - put.side_effect = [(0, True)] - disk.fetch = fetch - io_error = IOError() - io_error.errno = errno.EACCES - fetch.side_effect = io_error - - with mock.patch.object(cache, '_disk', disk): - with pytest.raises(IOError): - cache.get(0) - - def test_pop(cache): assert cache.incr('alpha') == 1 assert cache.pop('alpha') == 1 @@ -396,25 +376,6 @@ def test_pop_ioerror(cache): assert cache.pop(0) is None -def test_pop_ioerror_eacces(cache): - assert cache.set(0, 0) - - disk = mock.Mock() - put = mock.Mock() - fetch = mock.Mock() - - disk.put = put - put.side_effect = [(0, True)] - disk.fetch = fetch - io_error = IOError() - io_error.errno = errno.EACCES - fetch.side_effect = io_error - - with mock.patch.object(cache, '_disk', disk): - with pytest.raises(IOError): - cache.pop(0) - - def test_delete(cache): cache[0] = 0 assert cache.delete(0) @@ -591,29 +552,6 @@ def test_least_frequently_used(cache): assert len(cache.check()) == 0 -def test_filename_error(cache): - func = mock.Mock(side_effect=OSError(errno.EACCES)) - - with mock.patch('os.makedirs', func): - with pytest.raises(OSError): - cache._disk.filename() - - -def test_remove_error(cache): - func = mock.Mock(side_effect=OSError(errno.EACCES)) - - try: - with mock.patch('os.remove', func): - cache._disk.remove('ab/cd/efg.val') - except OSError: - pass - else: - if os.name == 'nt': - pass # File delete errors ignored on Windows. - else: - raise Exception('test_remove_error failed') - - def test_check(cache): blob = b'a' * 2 ** 20 keys = (0, 1, 1234, 56.78, u'hello', b'world', None) @@ -1028,44 +966,6 @@ def test_peek_ioerror(cache): assert value == 0 -def test_pull_ioerror_eacces(cache): - assert cache.push(0) == 500000000000000 - - disk = mock.Mock() - put = mock.Mock() - fetch = mock.Mock() - - disk.put = put - put.side_effect = [(0, True)] - disk.fetch = fetch - io_error = IOError() - io_error.errno = errno.EACCES - fetch.side_effect = io_error - - with mock.patch.object(cache, '_disk', disk): - with pytest.raises(IOError): - cache.pull() - - -def test_peek_ioerror_eacces(cache): - assert cache.push(0) == 500000000000000 - - disk = mock.Mock() - put = mock.Mock() - fetch = mock.Mock() - - disk.put = put - put.side_effect = [(0, True)] - disk.fetch = fetch - io_error = IOError() - io_error.errno = errno.EACCES - fetch.side_effect = io_error - - with mock.patch.object(cache, '_disk', disk): - with pytest.raises(IOError): - cache.peek() - - def test_peekitem_extras(cache): with pytest.raises(KeyError): cache.peekitem() @@ -1117,27 +1017,6 @@ def test_peekitem_ioerror(cache): assert value == 2 -def test_peekitem_ioerror_eacces(cache): - assert cache.set('a', 0) - assert cache.set('b', 1) - assert cache.set('c', 2) - - disk = mock.Mock() - put = mock.Mock() - fetch = mock.Mock() - - disk.put = put - put.side_effect = [(0, True)] - disk.fetch = fetch - io_error = IOError() - io_error.errno = errno.EACCES - fetch.side_effect = io_error - - with mock.patch.object(cache, '_disk', disk): - with pytest.raises(IOError): - cache.peekitem() - - def test_iterkeys(cache): assert list(cache.iterkeys()) == [] From 2f18867705a536d8a6f54404856f029923204f08 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 6 Sep 2021 22:40:36 -0700 Subject: [PATCH 067/126] Test Cache.memoize() with typed kwargs --- tests/test_core.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_core.py b/tests/test_core.py index a113443..518f99e 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1338,3 +1338,10 @@ def fibrec(num): assert hits2 == (hits1 + count) assert misses2 == misses1 + + +def test_memoize_kwargs(cache): + @cache.memoize(typed=True) + def foo(*args, **kwargs): + return args, kwargs + assert foo(1, 2, 3, a=4, b=5) == ((1, 2, 3), {'a': 4, 'b': 5}) From 094b873e1a780004d6b07dd3ebbb216898d803e4 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 6 Sep 2021 22:40:53 -0700 Subject: [PATCH 068/126] Test JSONDisk.get by iterating cache --- tests/test_core.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_core.py b/tests/test_core.py index 518f99e..efe3e24 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -89,6 +89,9 @@ def test_custom_disk(): for value in values: assert cache[value] == value + for key, value in zip(cache, values): + assert key == value + shutil.rmtree(cache.directory, ignore_errors=True) From b92f2fd3abaf9c30a921ee7d287f573f2d6546a6 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 6 Sep 2021 22:41:07 -0700 Subject: [PATCH 069/126] Increase coverage to 97% --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 04b7382..926a2b3 100644 --- a/tox.ini +++ b/tox.ini @@ -79,7 +79,7 @@ line_length = 79 addopts= -n auto --cov-branch - --cov-fail-under=96 + --cov-fail-under=97 --cov-report=term-missing --cov=diskcache --doctest-glob="*.rst" From f9503321a78ecffabac670bdd147bb6026af78fa Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 6 Sep 2021 22:46:16 -0700 Subject: [PATCH 070/126] Add test for cleaning up dirs --- tests/test_core.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_core.py b/tests/test_core.py index efe3e24..cb44da6 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1348,3 +1348,16 @@ def test_memoize_kwargs(cache): def foo(*args, **kwargs): return args, kwargs assert foo(1, 2, 3, a=4, b=5) == ((1, 2, 3), {'a': 4, 'b': 5}) + + +def test_cleanup_dirs(cache): + value = b'\0' * 2**20 + start_count = len(os.listdir(cache.directory)) + for i in range(10): + cache[i] = value + set_count = len(os.listdir(cache.directory)) + assert set_count > start_count + for i in range(10): + del cache[i] + del_count = len(os.listdir(cache.directory)) + assert start_count == del_count From 587f00d97724ef9bd2f66fcfa4935fecfc1f86a3 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 6 Sep 2021 22:47:21 -0700 Subject: [PATCH 071/126] Add TODO for testing Disk._write --- tests/test_core.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_core.py b/tests/test_core.py index cb44da6..2d09f94 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1361,3 +1361,11 @@ def test_cleanup_dirs(cache): del cache[i] del_count = len(os.listdir(cache.directory)) assert start_count == del_count + + +# TODO: Add tests for Disk._write +# diskcache/core.py +## Disk._write +# - 234->exit +# - 242-246 +# - 255 From 28aa595a662754e88a0d2665b4a9c5eaeadd93fd Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 13 Sep 2021 20:25:34 -0700 Subject: [PATCH 072/126] Add tests for Disk._write --- diskcache/core.py | 2 -- tests/test_core.py | 11 +++++------ 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/diskcache/core.py b/diskcache/core.py index 332276a..b836e84 100644 --- a/diskcache/core.py +++ b/diskcache/core.py @@ -252,8 +252,6 @@ def _write(self, full_path, iterator, mode, encoding=None): writer.write(chunk) return size - break - def fetch(self, mode, filename, value, read): """Convert fields `mode`, `filename`, and `value` from Cache table to value. diff --git a/tests/test_core.py b/tests/test_core.py index 2d09f94..41cfe51 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1363,9 +1363,8 @@ def test_cleanup_dirs(cache): assert start_count == del_count -# TODO: Add tests for Disk._write -# diskcache/core.py -## Disk._write -# - 234->exit -# - 242-246 -# - 255 +def test_disk_write_os_error(cache): + func = mock.Mock(side_effect=[OSError] * 10) + with mock.patch('diskcache.core.open', func): + with pytest.raises(OSError): + cache[0] = '\0' * 2**20 From a2e461ad93f7fa99a0d861db614cea2f7e521a6c Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 13 Sep 2021 20:38:35 -0700 Subject: [PATCH 073/126] Add a pragma "no cover" statements and increase threshold to 98 --- diskcache/__init__.py | 2 +- diskcache/djangocache.py | 2 +- tox.ini | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/diskcache/__init__.py b/diskcache/__init__.py index 428eb30..b5a6218 100644 --- a/diskcache/__init__.py +++ b/diskcache/__init__.py @@ -57,7 +57,7 @@ from .djangocache import DjangoCache # noqa __all__.append('DjangoCache') -except Exception: # pylint: disable=broad-except +except Exception: # pylint: disable=broad-except # pragma: no cover # Django not installed or not setup so ignore. pass diff --git a/diskcache/djangocache.py b/diskcache/djangocache.py index 44f673d..bf9f4d4 100644 --- a/diskcache/djangocache.py +++ b/diskcache/djangocache.py @@ -6,7 +6,7 @@ try: from django.core.cache.backends.base import DEFAULT_TIMEOUT -except ImportError: +except ImportError: # pragma: no cover # For older versions of Django simply use 300 seconds. DEFAULT_TIMEOUT = 300 diff --git a/tox.ini b/tox.ini index 926a2b3..104a084 100644 --- a/tox.ini +++ b/tox.ini @@ -79,7 +79,7 @@ line_length = 79 addopts= -n auto --cov-branch - --cov-fail-under=97 + --cov-fail-under=98 --cov-report=term-missing --cov=diskcache --doctest-glob="*.rst" From ab1484daf149039a468ed087b5073198bc4f21c1 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 13 Sep 2021 20:42:06 -0700 Subject: [PATCH 074/126] Blue fixes (mostly docstring triple quotes) --- diskcache/__init__.py | 1 - diskcache/cli.py | 2 +- diskcache/core.py | 29 ++++++++++++++--------------- diskcache/djangocache.py | 14 +++++++------- diskcache/fanout.py | 12 ++++++------ diskcache/persistent.py | 13 ++++++------- diskcache/recipes.py | 29 ++++++++++++++--------------- tests/benchmark_core.py | 1 - tests/benchmark_djangocache.py | 2 -- tests/benchmark_glob.py | 2 +- tests/benchmark_incr.py | 5 ++--- tests/benchmark_kv_store.py | 1 - tests/issue_109.py | 1 - tests/issue_85.py | 1 - tests/plot.py | 9 ++++----- tests/plot_early_recompute.py | 5 ++--- tests/stress_test_core.py | 10 +++++----- tests/stress_test_fanout.py | 10 +++++----- tests/test_core.py | 7 ++++--- tests/test_deque.py | 2 +- tests/test_djangocache.py | 4 ++-- tests/test_fanout.py | 2 +- tests/test_index.py | 2 +- tests/test_recipes.py | 2 +- tests/utils.py | 6 +++--- 25 files changed, 80 insertions(+), 92 deletions(-) diff --git a/diskcache/__init__.py b/diskcache/__init__.py index b5a6218..934a3a7 100644 --- a/diskcache/__init__.py +++ b/diskcache/__init__.py @@ -3,7 +3,6 @@ ======================= The :doc:`tutorial` provides a helpful walkthrough of most methods. - """ from .core import ( diff --git a/diskcache/cli.py b/diskcache/cli.py index 44bffeb..6a39f60 100644 --- a/diskcache/cli.py +++ b/diskcache/cli.py @@ -1 +1 @@ -"Command line interface to disk cache." +"""Command line interface to disk cache.""" diff --git a/diskcache/core.py b/diskcache/core.py index b836e84..4cbf67f 100644 --- a/diskcache/core.py +++ b/diskcache/core.py @@ -1,5 +1,4 @@ """Core disk and file backed cache API. - """ import codecs @@ -22,12 +21,12 @@ def full_name(func): - "Return full name of `func` by adding the module and function name." + """Return full name of `func` by adding the module and function name.""" return func.__module__ + '.' + func.__qualname__ class Constant(tuple): - "Pretty display of immutable constant." + """Pretty display of immutable constant.""" def __new__(cls, name): return tuple.__new__(cls, (name,)) @@ -102,7 +101,7 @@ def __repr__(self): class Disk: - "Cache key and value serialization for SQLite database and files." + """Cache key and value serialization for SQLite database and files.""" def __init__(self, directory, min_file_size=0, pickle_protocol=0): """Initialize disk instance. @@ -333,7 +332,7 @@ def remove(self, file_path): class JSONDisk(Disk): - "Cache key and value using JSON serialization with zlib compression." + """Cache key and value using JSON serialization with zlib compression.""" def __init__(self, directory, compress_level=1, **kwargs): """Initialize JSON disk instance. @@ -374,15 +373,15 @@ def fetch(self, mode, filename, value, read): class Timeout(Exception): - "Database timeout expired." + """Database timeout expired.""" class UnknownFileWarning(UserWarning): - "Warning used by Cache.check for unknown files." + """Warning used by Cache.check for unknown files.""" class EmptyDirWarning(UserWarning): - "Warning used by Cache.check for empty directories." + """Warning used by Cache.check for empty directories.""" def args_to_key(base, args, kwargs, typed): @@ -414,7 +413,7 @@ def args_to_key(base, args, kwargs, typed): class Cache: - "Disk and file backed cache." + """Disk and file backed cache.""" def __init__(self, directory=None, timeout=60, disk=Disk, **settings): """Initialize cache instance. @@ -1859,12 +1858,12 @@ def memoize(self, name=None, typed=False, expire=None, tag=None): raise TypeError('name cannot be callable') def decorator(func): - "Decorator created by memoize() for callable `func`." + """Decorator created by memoize() for callable `func`.""" base = (full_name(func),) if name is None else (name,) @ft.wraps(func) def wrapper(*args, **kwargs): - "Wrapper for callable to cache arguments and return values." + """Wrapper for callable to cache arguments and return values.""" key = wrapper.__cache_key__(*args, **kwargs) result = self.get(key, default=ENOVAL, retry=True) @@ -1876,7 +1875,7 @@ def wrapper(*args, **kwargs): return result def __cache_key__(*args, **kwargs): - "Make key for cache given function arguments." + """Make key for cache given function arguments.""" return args_to_key(base, args, kwargs, typed) wrapper.__cache_key__ = __cache_key__ @@ -2291,13 +2290,13 @@ def _iter(self, ascending=True): yield _disk_get(key, raw) def __iter__(self): - "Iterate keys in cache including expired items." + """Iterate keys in cache including expired items.""" iterator = self._iter() next(iterator) return iterator def __reversed__(self): - "Reverse iterate keys in cache including expired items." + """Reverse iterate keys in cache including expired items.""" iterator = self._iter(ascending=False) next(iterator) return iterator @@ -2355,7 +2354,7 @@ def __exit__(self, *exception): self.close() def __len__(self): - "Count of items in cache including expired items." + """Count of items in cache including expired items.""" return self.reset('count') def __getstate__(self): diff --git a/diskcache/djangocache.py b/diskcache/djangocache.py index bf9f4d4..449f3a0 100644 --- a/diskcache/djangocache.py +++ b/diskcache/djangocache.py @@ -1,4 +1,4 @@ -"Django-compatible disk and file backed cache." +"""Django-compatible disk and file backed cache.""" from functools import wraps @@ -15,7 +15,7 @@ class DjangoCache(BaseCache): - "Django-compatible disk and file backed cache." + """Django-compatible disk and file backed cache.""" def __init__(self, directory, params): """Initialize DjangoCache instance. @@ -344,11 +344,11 @@ def cull(self): return self._cache.cull() def clear(self): - "Remove *all* values from the cache at once." + """Remove *all* values from the cache at once.""" return self._cache.clear() def close(self, **kwargs): - "Close the cache connection." + """Close the cache connection.""" # pylint: disable=unused-argument self._cache.close() @@ -415,12 +415,12 @@ def memoize( raise TypeError('name cannot be callable') def decorator(func): - "Decorator created by memoize() for callable `func`." + """Decorator created by memoize() for callable `func`.""" base = (full_name(func),) if name is None else (name,) @wraps(func) def wrapper(*args, **kwargs): - "Wrapper for callable to cache arguments and return values." + """Wrapper for callable to cache arguments and return values.""" key = wrapper.__cache_key__(*args, **kwargs) result = self.get(key, ENOVAL, version, retry=True) @@ -444,7 +444,7 @@ def wrapper(*args, **kwargs): return result def __cache_key__(*args, **kwargs): - "Make key for cache given function arguments." + """Make key for cache given function arguments.""" return args_to_key(base, args, kwargs, typed) wrapper.__cache_key__ = __cache_key__ diff --git a/diskcache/fanout.py b/diskcache/fanout.py index 99384a0..dc5240c 100644 --- a/diskcache/fanout.py +++ b/diskcache/fanout.py @@ -1,4 +1,4 @@ -"Fanout cache automatically shards keys and values." +"""Fanout cache automatically shards keys and values.""" import contextlib as cl import functools @@ -14,7 +14,7 @@ class FanoutCache: - "Cache that shards keys and values." + """Cache that shards keys and values.""" def __init__( self, directory=None, shards=8, timeout=0.010, disk=Disk, **settings @@ -512,7 +512,7 @@ def volume(self): return sum(shard.volume() for shard in self._shards) def close(self): - "Close database connection." + """Close database connection.""" for shard in self._shards: shard.close() self._caches.clear() @@ -532,17 +532,17 @@ def __setstate__(self, state): self.__init__(*state) def __iter__(self): - "Iterate keys in cache including expired items." + """Iterate keys in cache including expired items.""" iterators = (iter(shard) for shard in self._shards) return it.chain.from_iterable(iterators) def __reversed__(self): - "Reverse iterate keys in cache including expired items." + """Reverse iterate keys in cache including expired items.""" iterators = (reversed(shard) for shard in reversed(self._shards)) return it.chain.from_iterable(iterators) def __len__(self): - "Count of items in cache including expired items." + """Count of items in cache including expired items.""" return sum(len(shard) for shard in self._shards) def reset(self, key, value=ENOVAL): diff --git a/diskcache/persistent.py b/diskcache/persistent.py index 44f9cc7..89c3899 100644 --- a/diskcache/persistent.py +++ b/diskcache/persistent.py @@ -1,5 +1,4 @@ """Persistent Data Types - """ import operator as op @@ -18,10 +17,10 @@ def _make_compare(seq_op, doc): - "Make compare method with Sequence semantics." + """Make compare method with Sequence semantics.""" def compare(self, that): - "Compare method for deque and sequence." + """Compare method for deque and sequence.""" if not isinstance(that, Sequence): return NotImplemented @@ -117,12 +116,12 @@ def fromcache(cls, cache, iterable=()): @property def cache(self): - "Cache used by deque." + """Cache used by deque.""" return self._cache @property def directory(self): - "Directory path where deque is stored." + """Directory path where deque is stored.""" return self._cache.directory def _index(self, index, func): @@ -699,12 +698,12 @@ def fromcache(cls, cache, *args, **kwargs): @property def cache(self): - "Cache used by index." + """Cache used by index.""" return self._cache @property def directory(self): - "Directory path where items are stored." + """Directory path where items are stored.""" return self._cache.directory def __getitem__(self, key): diff --git a/diskcache/recipes.py b/diskcache/recipes.py index 8f7bd32..0c02dd7 100644 --- a/diskcache/recipes.py +++ b/diskcache/recipes.py @@ -1,5 +1,4 @@ """Disk Cache Recipes - """ import functools @@ -40,7 +39,7 @@ def __init__(self, cache, key, expire=None, tag=None): self._tag = tag def add(self, value): - "Add `value` to average." + """Add `value` to average.""" with self._cache.transact(retry=True): total, count = self._cache.get(self._key, default=(0.0, 0)) total += value @@ -53,12 +52,12 @@ def add(self, value): ) def get(self): - "Get current average or return `None` if count equals zero." + """Get current average or return `None` if count equals zero.""" total, count = self._cache.get(self._key, default=(0.0, 0), retry=True) return None if count == 0 else total / count def pop(self): - "Return current average and delete key." + """Return current average and delete key.""" total, count = self._cache.pop(self._key, default=(0.0, 0), retry=True) return None if count == 0 else total / count @@ -83,7 +82,7 @@ def __init__(self, cache, key, expire=None, tag=None): self._tag = tag def acquire(self): - "Acquire lock using spin-lock algorithm." + """Acquire lock using spin-lock algorithm.""" while True: added = self._cache.add( self._key, @@ -97,11 +96,11 @@ def acquire(self): time.sleep(0.001) def release(self): - "Release lock by deleting key." + """Release lock by deleting key.""" self._cache.delete(self._key, retry=True) def locked(self): - "Return true if the lock is acquired." + """Return true if the lock is acquired.""" return self._key in self._cache def __enter__(self): @@ -137,7 +136,7 @@ def __init__(self, cache, key, expire=None, tag=None): self._tag = tag def acquire(self): - "Acquire lock by incrementing count using spin-lock algorithm." + """Acquire lock by incrementing count using spin-lock algorithm.""" pid = os.getpid() tid = threading.get_ident() pid_tid = '{}-{}'.format(pid, tid) @@ -156,7 +155,7 @@ def acquire(self): time.sleep(0.001) def release(self): - "Release lock by decrementing count." + """Release lock by decrementing count.""" pid = os.getpid() tid = threading.get_ident() pid_tid = '{}-{}'.format(pid, tid) @@ -206,7 +205,7 @@ def __init__(self, cache, key, value=1, expire=None, tag=None): self._tag = tag def acquire(self): - "Acquire semaphore by decrementing value using spin-lock algorithm." + """Acquire semaphore by decrementing value using spin-lock algorithm.""" while True: with self._cache.transact(retry=True): value = self._cache.get(self._key, default=self._value) @@ -221,7 +220,7 @@ def acquire(self): time.sleep(0.001) def release(self): - "Release semaphore by incrementing value." + """Release semaphore by incrementing value.""" with self._cache.transact(retry=True): value = self._cache.get(self._key, default=self._value) assert self._value > value, 'cannot release un-acquired semaphore' @@ -396,11 +395,11 @@ def memoize_stampede(cache, expire, name=None, typed=False, tag=None, beta=1): """ # Caution: Nearly identical code exists in Cache.memoize def decorator(func): - "Decorator created by memoize call for callable." + """Decorator created by memoize call for callable.""" base = (full_name(func),) if name is None else (name,) def timer(*args, **kwargs): - "Time execution of `func` and return result and time delta." + """Time execution of `func` and return result and time delta.""" start = time.time() result = func(*args, **kwargs) delta = time.time() - start @@ -408,7 +407,7 @@ def timer(*args, **kwargs): @functools.wraps(func) def wrapper(*args, **kwargs): - "Wrapper for callable to cache arguments and return values." + """Wrapper for callable to cache arguments and return values.""" key = wrapper.__cache_key__(*args, **kwargs) pair, expire_time = cache.get( key, @@ -459,7 +458,7 @@ def recompute(): return pair[0] def __cache_key__(*args, **kwargs): - "Make key for cache given function arguments." + """Make key for cache given function arguments.""" return args_to_key(base, args, kwargs, typed) wrapper.__cache_key__ = __cache_key__ diff --git a/tests/benchmark_core.py b/tests/benchmark_core.py index 282ce2a..7d64595 100644 --- a/tests/benchmark_core.py +++ b/tests/benchmark_core.py @@ -3,7 +3,6 @@ $ export PYTHONPATH=/Users/grantj/repos/python-diskcache $ python tests/benchmark_core.py -p 1 > tests/timings_core_p1.txt $ python tests/benchmark_core.py -p 8 > tests/timings_core_p8.txt - """ import collections as co diff --git a/tests/benchmark_djangocache.py b/tests/benchmark_djangocache.py index 9dbbcd7..61a80bf 100644 --- a/tests/benchmark_djangocache.py +++ b/tests/benchmark_djangocache.py @@ -2,8 +2,6 @@ $ export PYTHONPATH=/Users/grantj/repos/python-diskcache $ python tests/benchmark_djangocache.py > tests/timings_djangocache.txt - - """ import collections as co diff --git a/tests/benchmark_glob.py b/tests/benchmark_glob.py index 9c23104..7f0bf7c 100644 --- a/tests/benchmark_glob.py +++ b/tests/benchmark_glob.py @@ -1,4 +1,4 @@ -"Benchmark glob.glob1 as used by django.core.cache.backends.filebased." +"""Benchmark glob.glob1 as used by django.core.cache.backends.filebased.""" import os import os.path as op diff --git a/tests/benchmark_incr.py b/tests/benchmark_incr.py index 9c8e2fa..4f758aa 100644 --- a/tests/benchmark_incr.py +++ b/tests/benchmark_incr.py @@ -1,5 +1,4 @@ """Benchmark cache.incr method. - """ import json @@ -16,7 +15,7 @@ def worker(num): - "Rapidly increment key and time operation." + """Rapidly increment key and time operation.""" time.sleep(0.1) # Let other workers start. cache = dc.Cache('tmp') @@ -33,7 +32,7 @@ def worker(num): def main(): - "Run workers and print percentile results." + """Run workers and print percentile results.""" shutil.rmtree('tmp', ignore_errors=True) processes = [ diff --git a/tests/benchmark_kv_store.py b/tests/benchmark_kv_store.py index e141a36..7015470 100644 --- a/tests/benchmark_kv_store.py +++ b/tests/benchmark_kv_store.py @@ -1,7 +1,6 @@ """Benchmarking Key-Value Stores $ python -m IPython tests/benchmark_kv_store.py - """ from IPython import get_ipython diff --git a/tests/issue_109.py b/tests/issue_109.py index c10a81f..a649c58 100644 --- a/tests/issue_109.py +++ b/tests/issue_109.py @@ -1,5 +1,4 @@ """Benchmark for Issue #109 - """ import time diff --git a/tests/issue_85.py b/tests/issue_85.py index 723406b..cb8789b 100644 --- a/tests/issue_85.py +++ b/tests/issue_85.py @@ -2,7 +2,6 @@ $ export PYTHONPATH=`pwd` $ python tests/issue_85.py - """ import collections diff --git a/tests/plot.py b/tests/plot.py index 2138659..fcac0bc 100644 --- a/tests/plot.py +++ b/tests/plot.py @@ -2,7 +2,6 @@ $ export PYTHONPATH=/Users/grantj/repos/python-diskcache $ python tests/plot.py --show tests/timings_core_p1.txt - """ import argparse @@ -14,7 +13,7 @@ def parse_timing(timing, limit): - "Parse timing." + """Parse timing.""" if timing.endswith('ms'): value = float(timing[:-2]) * 1e-3 elif timing.endswith('us'): @@ -26,12 +25,12 @@ def parse_timing(timing, limit): def parse_row(row, line): - "Parse row." + """Parse row.""" return [val.strip() for val in row.match(line).groups()] def parse_data(infile): - "Parse data from `infile`." + """Parse data from `infile`.""" blocks = re.compile(' '.join(['=' * 9] * 8)) dashes = re.compile('^-{79}$') title = re.compile('^Timings for (.*)$') @@ -83,7 +82,7 @@ def parse_data(infile): def make_plot(data, action, save=False, show=False, limit=0.005): - "Make plot." + """Make plot.""" fig, ax = plt.subplots(figsize=(8, 10)) colors = ['#ff7f00', '#377eb8', '#4daf4a', '#984ea3', '#e41a1c'] width = 0.15 diff --git a/tests/plot_early_recompute.py b/tests/plot_early_recompute.py index e58f580..1508c45 100644 --- a/tests/plot_early_recompute.py +++ b/tests/plot_early_recompute.py @@ -1,5 +1,4 @@ """Early Recomputation Measurements - """ import functools as ft @@ -61,14 +60,14 @@ def repeat(num): def frange(start, stop, step=1e-3): - "Generator for floating point values from `start` to `stop` by `step`." + """Generator for floating point values from `start` to `stop` by `step`.""" while start < stop: yield start start += step def plot(option, filename, cache_times, worker_times): - "Plot concurrent workers and latency." + """Plot concurrent workers and latency.""" import matplotlib.pyplot as plt fig, (workers, latency) = plt.subplots(2, sharex=True) diff --git a/tests/stress_test_core.py b/tests/stress_test_core.py index 6fd0991..c30fa3f 100644 --- a/tests/stress_test_core.py +++ b/tests/stress_test_core.py @@ -1,4 +1,4 @@ -"Stress test diskcache.core.Cache." +"""Stress test diskcache.core.Cache.""" import collections as co import multiprocessing as mp @@ -292,22 +292,22 @@ def stress_test( def stress_test_lru(): - "Stress test least-recently-used eviction policy." + """Stress test least-recently-used eviction policy.""" stress_test(eviction_policy=u'least-recently-used') def stress_test_lfu(): - "Stress test least-frequently-used eviction policy." + """Stress test least-frequently-used eviction policy.""" stress_test(eviction_policy=u'least-frequently-used') def stress_test_none(): - "Stress test 'none' eviction policy." + """Stress test 'none' eviction policy.""" stress_test(eviction_policy=u'none') def stress_test_mp(): - "Stress test multiple threads and processes." + """Stress test multiple threads and processes.""" stress_test(processes=4, threads=4) diff --git a/tests/stress_test_fanout.py b/tests/stress_test_fanout.py index 58708c9..d3b67e3 100644 --- a/tests/stress_test_fanout.py +++ b/tests/stress_test_fanout.py @@ -1,4 +1,4 @@ -"Stress test diskcache.core.Cache." +"""Stress test diskcache.core.Cache.""" import multiprocessing as mp import os @@ -283,22 +283,22 @@ def stress_test( def stress_test_lru(): - "Stress test least-recently-used eviction policy." + """Stress test least-recently-used eviction policy.""" stress_test(eviction_policy=u'least-recently-used') def stress_test_lfu(): - "Stress test least-frequently-used eviction policy." + """Stress test least-frequently-used eviction policy.""" stress_test(eviction_policy=u'least-frequently-used') def stress_test_none(): - "Stress test 'none' eviction policy." + """Stress test 'none' eviction policy.""" stress_test(eviction_policy=u'none') def stress_test_mp(): - "Stress test multiple threads and processes." + """Stress test multiple threads and processes.""" stress_test(processes=4, threads=4) diff --git a/tests/test_core.py b/tests/test_core.py index 41cfe51..c1e7a4a 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1,4 +1,4 @@ -"Test diskcache.core.Cache." +"""Test diskcache.core.Cache.""" import errno import hashlib @@ -1347,11 +1347,12 @@ def test_memoize_kwargs(cache): @cache.memoize(typed=True) def foo(*args, **kwargs): return args, kwargs + assert foo(1, 2, 3, a=4, b=5) == ((1, 2, 3), {'a': 4, 'b': 5}) def test_cleanup_dirs(cache): - value = b'\0' * 2**20 + value = b'\0' * 2 ** 20 start_count = len(os.listdir(cache.directory)) for i in range(10): cache[i] = value @@ -1367,4 +1368,4 @@ def test_disk_write_os_error(cache): func = mock.Mock(side_effect=[OSError] * 10) with mock.patch('diskcache.core.open', func): with pytest.raises(OSError): - cache[0] = '\0' * 2**20 + cache[0] = '\0' * 2 ** 20 diff --git a/tests/test_deque.py b/tests/test_deque.py index 8113dfe..add7714 100644 --- a/tests/test_deque.py +++ b/tests/test_deque.py @@ -1,4 +1,4 @@ -"Test diskcache.persistent.Deque." +"""Test diskcache.persistent.Deque.""" import pickle import shutil diff --git a/tests/test_djangocache.py b/tests/test_djangocache.py index 2c216fe..cdaf101 100644 --- a/tests/test_djangocache.py +++ b/tests/test_djangocache.py @@ -100,7 +100,7 @@ class UnpicklableType(object): def custom_key_func(key, key_prefix, version): - "A customized cache key function" + """A customized cache key function""" return 'CUSTOM-' + '-'.join([key_prefix, str(version), key]) @@ -921,7 +921,7 @@ def __getstate__(self): ) ) class DiskCacheTests(BaseCacheTests, TestCase): - "Specific test cases for diskcache.DjangoCache." + """Specific test cases for diskcache.DjangoCache.""" def setUp(self): super().setUp() diff --git a/tests/test_fanout.py b/tests/test_fanout.py index 8918af3..f212fac 100644 --- a/tests/test_fanout.py +++ b/tests/test_fanout.py @@ -1,4 +1,4 @@ -"Test diskcache.fanout.FanoutCache." +"""Test diskcache.fanout.FanoutCache.""" import collections as co import hashlib diff --git a/tests/test_index.py b/tests/test_index.py index 27639f7..742daf3 100644 --- a/tests/test_index.py +++ b/tests/test_index.py @@ -1,4 +1,4 @@ -"Test diskcache.persistent.Index." +"""Test diskcache.persistent.Index.""" import pickle import shutil diff --git a/tests/test_recipes.py b/tests/test_recipes.py index 3f330a6..ae74459 100644 --- a/tests/test_recipes.py +++ b/tests/test_recipes.py @@ -1,4 +1,4 @@ -"Test diskcache.recipes." +"""Test diskcache.recipes.""" import shutil import threading diff --git a/tests/utils.py b/tests/utils.py index 5b41ce9..38e5d33 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -32,7 +32,7 @@ def secs(value): def run(*args): - "Run command, print output, and return output." + """Run command, print output, and return output.""" print('utils$', *args) result = sp.check_output(args) print(result) @@ -40,7 +40,7 @@ def run(*args): def mount_ramdisk(size, path): - "Mount RAM disk at `path` with `size` in bytes." + """Mount RAM disk at `path` with `size` in bytes.""" sectors = size / 512 os.makedirs(path) @@ -53,7 +53,7 @@ def mount_ramdisk(size, path): def unmount_ramdisk(dev_path, path): - "Unmount RAM disk with `dev_path` and `path`." + """Unmount RAM disk with `dev_path` and `path`.""" run('umount', path) run('diskutil', 'eject', dev_path) run('rm', '-r', path) From 72bbd73d29184e095df248efd48c31eeeb8e992c Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 13 Sep 2021 20:58:43 -0700 Subject: [PATCH 075/126] Pylint fixes --- diskcache/core.py | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/diskcache/core.py b/diskcache/core.py index 4cbf67f..2e946ad 100644 --- a/diskcache/core.py +++ b/diskcache/core.py @@ -46,25 +46,25 @@ def __repr__(self): MODE_PICKLE = 4 DEFAULT_SETTINGS = { - u'statistics': 0, # False - u'tag_index': 0, # False - u'eviction_policy': u'least-recently-stored', - u'size_limit': 2 ** 30, # 1gb - u'cull_limit': 10, - u'sqlite_auto_vacuum': 1, # FULL - u'sqlite_cache_size': 2 ** 13, # 8,192 pages - u'sqlite_journal_mode': u'wal', - u'sqlite_mmap_size': 2 ** 26, # 64mb - u'sqlite_synchronous': 1, # NORMAL - u'disk_min_file_size': 2 ** 15, # 32kb - u'disk_pickle_protocol': pickle.HIGHEST_PROTOCOL, + 'statistics': 0, # False + 'tag_index': 0, # False + 'eviction_policy': 'least-recently-stored', + 'size_limit': 2 ** 30, # 1gb + 'cull_limit': 10, + 'sqlite_auto_vacuum': 1, # FULL + 'sqlite_cache_size': 2 ** 13, # 8,192 pages + 'sqlite_journal_mode': 'wal', + 'sqlite_mmap_size': 2 ** 26, # 64mb + 'sqlite_synchronous': 1, # NORMAL + 'disk_min_file_size': 2 ** 15, # 32kb + 'disk_pickle_protocol': pickle.HIGHEST_PROTOCOL, } METADATA = { - u'count': 0, - u'size': 0, - u'hits': 0, - u'misses': 0, + 'count': 0, + 'size': 0, + 'hits': 0, + 'misses': 0, } EVICTION_POLICY = { @@ -1194,7 +1194,7 @@ def get( try: value = self._disk.fetch(mode, filename, db_value, read) - except IOError as error: + except IOError: # Key was deleted before we could retrieve result. if self.statistics: sql(cache_miss) @@ -1314,7 +1314,7 @@ def pop( try: value = self._disk.fetch(mode, filename, db_value, False) - except IOError as error: + except IOError: # Key was deleted before we could retrieve result. return default finally: @@ -1582,7 +1582,7 @@ def pull( try: value = self._disk.fetch(mode, name, db_value, False) - except IOError as error: + except IOError: # Key was deleted before we could retrieve result. continue finally: @@ -1696,7 +1696,7 @@ def peek( try: value = self._disk.fetch(mode, name, db_value, False) - except IOError as error: + except IOError: # Key was deleted before we could retrieve result. continue finally: @@ -1777,7 +1777,7 @@ def peekitem(self, last=True, expire_time=False, tag=False, retry=False): try: value = self._disk.fetch(mode, name, db_value, False) - except IOError as error: + except IOError: # Key was deleted before we could retrieve result. continue break @@ -1911,7 +1911,7 @@ def check(self, fix=False, retry=False): rows = sql('PRAGMA integrity_check').fetchall() - if len(rows) != 1 or rows[0][0] != u'ok': + if len(rows) != 1 or rows[0][0] != 'ok': for (message,) in rows: warnings.warn(message) From 3e87128d4154acf90bd64c6630b1200108dc1ed2 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 13 Sep 2021 21:04:34 -0700 Subject: [PATCH 076/126] Disable no-self-use in Disk._write --- diskcache/core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/diskcache/core.py b/diskcache/core.py index 2e946ad..251c5a2 100644 --- a/diskcache/core.py +++ b/diskcache/core.py @@ -228,6 +228,7 @@ def store(self, value, read, key=UNKNOWN): return len(result), MODE_PICKLE, filename, None def _write(self, full_path, iterator, mode, encoding=None): + # pylint: disable=no-self-use full_dir, _ = op.split(full_path) for count in range(1, 11): From fbc537a7138330652d0d42aa09caa5531746b302 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 13 Sep 2021 21:51:30 -0700 Subject: [PATCH 077/126] Add `ignore` to memoize() --- diskcache/core.py | 10 +++++++--- diskcache/djangocache.py | 4 +++- diskcache/persistent.py | 5 +++-- diskcache/recipes.py | 5 +++-- tests/test_core.py | 13 +++++++++++++ 5 files changed, 29 insertions(+), 8 deletions(-) diff --git a/diskcache/core.py b/diskcache/core.py index 251c5a2..4035293 100644 --- a/diskcache/core.py +++ b/diskcache/core.py @@ -385,19 +385,22 @@ class EmptyDirWarning(UserWarning): """Warning used by Cache.check for empty directories.""" -def args_to_key(base, args, kwargs, typed): +def args_to_key(base, args, kwargs, typed, ignore): """Create cache key out of function arguments. :param tuple base: base of key :param tuple args: function arguments :param dict kwargs: function keyword arguments :param bool typed: include types in cache key + :param set ignore: positional or keyword args to ignore :return: cache key tuple """ + args = tuple(arg for index, arg in enumerate(args) if index not in ignore) key = base + args if kwargs: + kwargs = {key: val for key, val in kwargs.items() if key not in ignore} key += (ENOVAL,) sorted_items = sorted(kwargs.items()) @@ -1792,7 +1795,7 @@ def peekitem(self, last=True, expire_time=False, tag=False, retry=False): else: return key, value - def memoize(self, name=None, typed=False, expire=None, tag=None): + def memoize(self, name=None, typed=False, expire=None, tag=None, ignore=()): """Memoizing cache decorator. Decorator to wrap callable with memoizing function using cache. @@ -1851,6 +1854,7 @@ def memoize(self, name=None, typed=False, expire=None, tag=None): :param float expire: seconds until arguments expire (default None, no expiry) :param str tag: text to associate with arguments (default None) + :param set ignore: positional or keyword args to ignore (default ()) :return: callable decorator """ @@ -1877,7 +1881,7 @@ def wrapper(*args, **kwargs): def __cache_key__(*args, **kwargs): """Make key for cache given function arguments.""" - return args_to_key(base, args, kwargs, typed) + return args_to_key(base, args, kwargs, typed, ignore) wrapper.__cache_key__ = __cache_key__ return wrapper diff --git a/diskcache/djangocache.py b/diskcache/djangocache.py index 449f3a0..347a613 100644 --- a/diskcache/djangocache.py +++ b/diskcache/djangocache.py @@ -373,6 +373,7 @@ def memoize( version=None, typed=False, tag=None, + ignore=(), ): """Memoizing cache decorator. @@ -407,6 +408,7 @@ def memoize( :param int version: key version number (default None, cache parameter) :param bool typed: cache different types separately (default False) :param str tag: text to associate with arguments (default None) + :param set ignore: positional or keyword args to ignore (default ()) :return: callable decorator """ @@ -445,7 +447,7 @@ def wrapper(*args, **kwargs): def __cache_key__(*args, **kwargs): """Make key for cache given function arguments.""" - return args_to_key(base, args, kwargs, typed) + return args_to_key(base, args, kwargs, typed, ignore) wrapper.__cache_key__ = __cache_key__ return wrapper diff --git a/diskcache/persistent.py b/diskcache/persistent.py index 89c3899..9b5939b 100644 --- a/diskcache/persistent.py +++ b/diskcache/persistent.py @@ -1096,7 +1096,7 @@ def __ne__(self, other): """ return not self == other - def memoize(self, name=None, typed=False): + def memoize(self, name=None, typed=False, ignore=()): """Memoizing cache decorator. Decorator to wrap callable with memoizing function using cache. @@ -1147,10 +1147,11 @@ def memoize(self, name=None, typed=False): :param str name: name given for callable (default None, automatic) :param bool typed: cache different types separately (default False) + :param set ignore: positional or keyword args to ignore (default ()) :return: callable decorator """ - return self._cache.memoize(name, typed) + return self._cache.memoize(name, typed, ignore) @contextmanager def transact(self): diff --git a/diskcache/recipes.py b/diskcache/recipes.py index 0c02dd7..b345560 100644 --- a/diskcache/recipes.py +++ b/diskcache/recipes.py @@ -337,7 +337,7 @@ def wrapper(*args, **kwargs): return decorator -def memoize_stampede(cache, expire, name=None, typed=False, tag=None, beta=1): +def memoize_stampede(cache, expire, name=None, typed=False, tag=None, beta=1, ignore=()): """Memoizing cache decorator with cache stampede protection. Cache stampedes are a type of system overload that can occur when parallel @@ -390,6 +390,7 @@ def memoize_stampede(cache, expire, name=None, typed=False, tag=None, beta=1): :param str name: name given for callable (default None, automatic) :param bool typed: cache different types separately (default False) :param str tag: text to associate with arguments (default None) + :param set ignore: positional or keyword args to ignore (default ()) :return: callable decorator """ @@ -459,7 +460,7 @@ def recompute(): def __cache_key__(*args, **kwargs): """Make key for cache given function arguments.""" - return args_to_key(base, args, kwargs, typed) + return args_to_key(base, args, kwargs, typed, ignore) wrapper.__cache_key__ = __cache_key__ return wrapper diff --git a/tests/test_core.py b/tests/test_core.py index c1e7a4a..0a1fca6 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1369,3 +1369,16 @@ def test_disk_write_os_error(cache): with mock.patch('diskcache.core.open', func): with pytest.raises(OSError): cache[0] = '\0' * 2 ** 20 + + +def test_memoize_ignore(cache): + + @cache.memoize(ignore={1, 'arg1'}) + def test(*args, **kwargs): + return args, kwargs + + cache.stats(enable=True) + assert test('a', 'b', 'c', arg0='d', arg1='e', arg2='f') + assert test('a', 'w', 'c', arg0='d', arg1='x', arg2='f') + assert test('a', 'y', 'c', arg0='d', arg1='z', arg2='f') + assert cache.stats() == (2, 1) From bd800aa069ad6bc15aa3f6fec84ae92d2f644de5 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 13 Sep 2021 21:52:05 -0700 Subject: [PATCH 078/126] Fixes for blue --- diskcache/core.py | 4 +++- diskcache/recipes.py | 4 +++- tests/test_core.py | 1 - 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/diskcache/core.py b/diskcache/core.py index 4035293..d7f707b 100644 --- a/diskcache/core.py +++ b/diskcache/core.py @@ -1795,7 +1795,9 @@ def peekitem(self, last=True, expire_time=False, tag=False, retry=False): else: return key, value - def memoize(self, name=None, typed=False, expire=None, tag=None, ignore=()): + def memoize( + self, name=None, typed=False, expire=None, tag=None, ignore=() + ): """Memoizing cache decorator. Decorator to wrap callable with memoizing function using cache. diff --git a/diskcache/recipes.py b/diskcache/recipes.py index b345560..b5af6dd 100644 --- a/diskcache/recipes.py +++ b/diskcache/recipes.py @@ -337,7 +337,9 @@ def wrapper(*args, **kwargs): return decorator -def memoize_stampede(cache, expire, name=None, typed=False, tag=None, beta=1, ignore=()): +def memoize_stampede( + cache, expire, name=None, typed=False, tag=None, beta=1, ignore=() +): """Memoizing cache decorator with cache stampede protection. Cache stampedes are a type of system overload that can occur when parallel diff --git a/tests/test_core.py b/tests/test_core.py index 0a1fca6..b3d3d66 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1372,7 +1372,6 @@ def test_disk_write_os_error(cache): def test_memoize_ignore(cache): - @cache.memoize(ignore={1, 'arg1'}) def test(*args, **kwargs): return args, kwargs From 606d8f2b12e8022e6d5b509d7c41439c4242043d Mon Sep 17 00:00:00 2001 From: Abhinav Omprakash <55880260+AbhinavOmprakash@users.noreply.github.com> Date: Fri, 11 Jun 2021 09:54:54 +0530 Subject: [PATCH 079/126] Fixes #201 added github repo to project_urls --- setup.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7f0561c..b29a463 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,12 @@ def run_tests(self): long_description=readme, author='Grant Jenks', author_email='contact@grantjenks.com', - url='http://www.grantjenks.com/docs/diskcache/', + url='http://www.grantjenks.com/docs/diskcache/', + project_urls = { + 'Documentation':'http://www.grantjenks.com/docs/diskcache/', + 'Source':'https://github.com/grantjenks/python-diskcache', + 'Tracker':'https://github.com/grantjenks/python-diskcache/issues', + 'Funding':'https://gumroad.com/l/diskcache',} license='Apache 2.0', packages=['diskcache'], tests_require=['tox'], From c22d3ee59ee28bd58aec59182d375b1eccd191f2 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 13 Sep 2021 22:03:57 -0700 Subject: [PATCH 080/126] Fixup formatting for project urls --- setup.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index b29a463..b49d9c8 100644 --- a/setup.py +++ b/setup.py @@ -29,12 +29,13 @@ def run_tests(self): long_description=readme, author='Grant Jenks', author_email='contact@grantjenks.com', - url='http://www.grantjenks.com/docs/diskcache/', - project_urls = { - 'Documentation':'http://www.grantjenks.com/docs/diskcache/', - 'Source':'https://github.com/grantjenks/python-diskcache', - 'Tracker':'https://github.com/grantjenks/python-diskcache/issues', - 'Funding':'https://gumroad.com/l/diskcache',} + url='http://www.grantjenks.com/docs/diskcache/', + project_urls={ + 'Documentation': 'http://www.grantjenks.com/docs/diskcache/', + 'Funding': 'https://gum.co/diskcache', + 'Source': 'https://github.com/grantjenks/python-diskcache', + 'Tracker': 'https://github.com/grantjenks/python-diskcache/issues', + }, license='Apache 2.0', packages=['diskcache'], tests_require=['tox'], From d55a50ee083784afa9c85e14e41c4a2d132f3111 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 13 Sep 2021 22:15:50 -0700 Subject: [PATCH 081/126] Stop using ENOVAL in args_to_key() --- diskcache/core.py | 3 +-- tests/test_core.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/diskcache/core.py b/diskcache/core.py index d7f707b..fb343be 100644 --- a/diskcache/core.py +++ b/diskcache/core.py @@ -397,11 +397,10 @@ def args_to_key(base, args, kwargs, typed, ignore): """ args = tuple(arg for index, arg in enumerate(args) if index not in ignore) - key = base + args + key = base + args + (None,) if kwargs: kwargs = {key: val for key, val in kwargs.items() if key not in ignore} - key += (ENOVAL,) sorted_items = sorted(kwargs.items()) for item in sorted_items: diff --git a/tests/test_core.py b/tests/test_core.py index b3d3d66..55ca962 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -92,6 +92,8 @@ def test_custom_disk(): for key, value in zip(cache, values): assert key == value + test_memoize_iter(cache) + shutil.rmtree(cache.directory, ignore_errors=True) @@ -1381,3 +1383,17 @@ def test(*args, **kwargs): assert test('a', 'w', 'c', arg0='d', arg1='x', arg2='f') assert test('a', 'y', 'c', arg0='d', arg1='z', arg2='f') assert cache.stats() == (2, 1) + + +def test_memoize_iter(cache): + @cache.memoize() + def test(*args, **kwargs): + return sum(args) + sum(kwargs.values()) + + cache.clear() + assert test(1, 2, 3) + assert test(a=1, b=2, c=3) + assert test(-1, 0, 1, a=1, b=2, c=3) + assert len(cache) == 3 + for key in cache: + assert cache[key] == 6 From b10b3866f4d64ea197e3fb40c0ebe25aa32900d6 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 13 Sep 2021 22:29:21 -0700 Subject: [PATCH 082/126] Add caveat about inconsistent pickles --- docs/tutorial.rst | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 58220b2..3191af6 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -863,8 +863,17 @@ protocol`_ is not used. Neither the `__hash__` nor `__eq__` methods are used for lookups. Instead lookups depend on the serialization method defined by :class:`Disk ` objects. For strings, bytes, integers, and floats, equality matches Python's definition. But large integers and all other -types will be converted to bytes using pickling and the bytes representation -will define equality. +types will be converted to bytes and the bytes representation will define +equality. + +The default :class:`diskcache.Disk` serialization uses pickling for both keys +and values. Unfortunately, pickling produces inconsistencies sometimes when +applied to container data types like tuples. Two equal tuples may serialize to +different bytes objects using pickle. The likelihood of differences is reduced +by using `pickletools.optimize` but still inconsistencies occur (`#54`_). The +inconsistent serialized pickle values is particularly problematic when applied +to the key in the cache. Consider using an alternative Disk type, like +:class:`JSONDisk `, for consistent serialization of keys. SQLite is used to synchronize database access between threads and processes and as such inherits all SQLite caveats. Most notably SQLite is `not recommended`_ @@ -898,6 +907,7 @@ does not account the size of directories themselves or other filesystem metadata. If directory count or size is a concern then consider implementing an alternative :class:`Disk `. +.. _`#54`: https://github.com/grantjenks/python-diskcache/issues/54 .. _`hash protocol`: https://docs.python.org/library/functions.html#hash .. _`not recommended`: https://www.sqlite.org/faq.html#q5 .. _`performs poorly`: https://www.pythonanywhere.com/forums/topic/1847/ From 3ad6c0e4365ef93e60a75aee75740e6551e279f9 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 13 Sep 2021 22:40:26 -0700 Subject: [PATCH 083/126] Bug Fix: Use "ignore" keyword argument with Index.memoize() --- diskcache/persistent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/diskcache/persistent.py b/diskcache/persistent.py index 9b5939b..c3d570b 100644 --- a/diskcache/persistent.py +++ b/diskcache/persistent.py @@ -1151,7 +1151,7 @@ def memoize(self, name=None, typed=False, ignore=()): :return: callable decorator """ - return self._cache.memoize(name, typed, ignore) + return self._cache.memoize(name, typed, ignore=ignore) @contextmanager def transact(self): From 4de3c0ed99dd2bcebffad8e6e0cb7a2885210a97 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Tue, 14 Sep 2021 14:07:00 -0700 Subject: [PATCH 084/126] Drop old Ubuntu from integration testing --- .github/workflows/integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 0a44f89..b1143a4 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -31,7 +31,7 @@ jobs: strategy: max-parallel: 8 matrix: - os: [ubuntu-latest, macos-latest, windows-latest, ubuntu-16.04] + os: [ubuntu-latest, macos-latest, windows-latest] python-version: [3.6, 3.7, 3.8, 3.9] steps: From 6ed012a655673c54468beb7ff00038443ed2595e Mon Sep 17 00:00:00 2001 From: artiom Date: Wed, 29 Sep 2021 10:57:36 +0100 Subject: [PATCH 085/126] docs: fix typo --- docs/tutorial.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 3191af6..1963635 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -570,7 +570,7 @@ access and editing at both front and back sides. :class:`Deque cross-thread and cross-process communication. :class:`Deque ` objects are also useful in scenarios where contents should remain persistent or limitations prohibit holding all items in memory at the same time. The deque -uses a fixed amout of memory regardless of the size or number of items stored +uses a fixed amount of memory regardless of the size or number of items stored inside it. Index @@ -603,7 +603,7 @@ interface. :class:`Index ` objects inherit all the benefits of cross-thread and cross-process communication. :class:`Index ` objects are also useful in scenarios where contents should remain persistent or limitations prohibit holding all items in memory at the same time. The index -uses a fixed amout of memory regardless of the size or number of items stored +uses a fixed amount of memory regardless of the size or number of items stored inside it. .. _tutorial-transactions: From 20f1d93a6e3852bac4289cb94e27585d9c23330c Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Wed, 29 Sep 2021 08:11:04 -0700 Subject: [PATCH 086/126] Disable consider-using-f-string --- .pylintrc | 1 + 1 file changed, 1 insertion(+) diff --git a/.pylintrc b/.pylintrc index 158fe34..6baa978 100644 --- a/.pylintrc +++ b/.pylintrc @@ -143,6 +143,7 @@ disable=print-statement, no-else-return, duplicate-code, inconsistent-return-statements, + consider-using-f-string, # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option From 7bfbce63ba127d601ddfbf2838539e34cad87616 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 29 Nov 2021 22:58:48 -0800 Subject: [PATCH 087/126] Support for Python 3.10 in testing (#238) * Add support for Python 3.10 * Update copyright to 2022 * Bump version to 5.3.0 * Add Python 3.10 to the README --- .github/workflows/integration.yml | 4 ++-- .github/workflows/release.yml | 2 +- LICENSE | 2 +- README.rst | 6 +++--- diskcache/__init__.py | 6 +++--- docs/conf.py | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index b1143a4..07a5650 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -16,7 +16,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: 3.9 + python-version: '3.10' - name: Install dependencies run: | pip install --upgrade pip @@ -32,7 +32,7 @@ jobs: max-parallel: 8 matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: [3.6, 3.7, 3.8, 3.9] + python-version: [3.6, 3.7, 3.8, 3.9, '3.10'] steps: - name: Set up Python ${{ matrix.python-version }} x64 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2a07787..1f89c14 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -21,7 +21,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: 3.9 + python-version: '3.10' - name: Install dependencies run: | diff --git a/LICENSE b/LICENSE index ca80a22..bb4cfb7 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright 2016-2021 Grant Jenks +Copyright 2016-2022 Grant Jenks Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the diff --git a/README.rst b/README.rst index c4aa8e9..eb06a6a 100644 --- a/README.rst +++ b/README.rst @@ -77,8 +77,8 @@ Features - Thread-safe and process-safe - Supports multiple eviction policies (LRU and LFU included) - Keys support "tag" metadata and eviction -- Developed on Python 3.9 -- Tested on CPython 3.6, 3.7, 3.8, 3.9 +- Developed on Python 3.10 +- Tested on CPython 3.6, 3.7, 3.8, 3.9, 3.10 - Tested on Linux, Mac OS X, and Windows - Tested using GitHub Actions @@ -387,7 +387,7 @@ Reference License ------- -Copyright 2016-2021 Grant Jenks +Copyright 2016-2022 Grant Jenks Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the diff --git a/diskcache/__init__.py b/diskcache/__init__.py index 934a3a7..c361ca9 100644 --- a/diskcache/__init__.py +++ b/diskcache/__init__.py @@ -61,8 +61,8 @@ pass __title__ = 'diskcache' -__version__ = '5.2.1' -__build__ = 0x050201 +__version__ = '5.3.0' +__build__ = 0x050300 __author__ = 'Grant Jenks' __license__ = 'Apache 2.0' -__copyright__ = 'Copyright 2016-2021 Grant Jenks' +__copyright__ = 'Copyright 2016-2022 Grant Jenks' diff --git a/docs/conf.py b/docs/conf.py index d725198..92ce1b9 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -20,7 +20,7 @@ # -- Project information ----------------------------------------------------- project = 'DiskCache' -copyright = '2021, Grant Jenks' +copyright = '2022, Grant Jenks' author = 'Grant Jenks' # The full version, including alpha/beta/rc tags From 78a5cc690e0aa53ad3537a72b2f52f3c161da1ae Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Thu, 30 Dec 2021 17:35:44 -0800 Subject: [PATCH 088/126] Update tests for Django 3.2 --- tests/test_djangocache.py | 266 +++++++++++++++++++++++++------------- 1 file changed, 176 insertions(+), 90 deletions(-) diff --git a/tests/test_djangocache.py b/tests/test_djangocache.py index cdaf101..36e1b01 100644 --- a/tests/test_djangocache.py +++ b/tests/test_djangocache.py @@ -1,5 +1,5 @@ # Most of this file was copied from: -# https://raw.githubusercontent.com/django/django/stable/2.2.x/tests/cache/tests.py +# https://raw.githubusercontent.com/django/django/stable/3.2.x/tests/cache/tests.py # Unit tests for cache framework # Uses whatever cache backend is set in the test settings file. @@ -9,12 +9,14 @@ import pickle import re import shutil +import sys import tempfile import threading import time import unittest import warnings -from unittest import mock +from pathlib import Path +from unittest import mock, skipIf from django.conf import settings from django.core import management, signals @@ -88,6 +90,10 @@ def __getstate__(self): raise pickle.PickleError() +def empty_response(request): + return HttpResponse() + + KEY_ERRORS_WITH_MEMCACHED_MSG = ( 'Cache key contains characters that will cause errors if used with ' 'memcached: %r' @@ -138,6 +144,14 @@ class BaseCacheTests: # A common set of tests to apply to all cache backends factory = RequestFactory() + # RemovedInDjango41Warning: python-memcached doesn't support .get() with + # default. + supports_get_with_default = True + + # Some clients raise custom exceptions when .incr() or .decr() are called + # with a non-integer value. + incr_decr_type_error = TypeError + def tearDown(self): cache.clear() @@ -146,11 +160,15 @@ def test_simple(self): cache.set('key', 'value') self.assertEqual(cache.get('key'), 'value') + def test_default_used_when_none_is_set(self): + """If None is cached, get() returns it instead of the default.""" + cache.set('key_default_none', None) + self.assertIsNone(cache.get('key_default_none', default='default')) + def test_add(self): # A key can be added to a cache - cache.add('addkey1', 'value') - result = cache.add('addkey1', 'newvalue') - self.assertFalse(result) + self.assertIs(cache.add('addkey1', 'value'), True) + self.assertIs(cache.add('addkey1', 'newvalue'), False) self.assertEqual(cache.get('addkey1'), 'value') def test_prefix(self): @@ -158,7 +176,7 @@ def test_prefix(self): cache.set('somekey', 'value') # should not be set in the prefixed cache - self.assertFalse(caches['prefix'].has_key('somekey')) + self.assertIs(caches['prefix'].has_key('somekey'), False) caches['prefix'].set('somekey', 'value2') @@ -180,28 +198,43 @@ def test_get_many(self): self.assertEqual( cache.get_many(iter(['a', 'b', 'e'])), {'a': 'a', 'b': 'b'} ) + cache.set_many({'x': None, 'y': 1}) + self.assertEqual(cache.get_many(['x', 'y']), {'x': None, 'y': 1}) def test_delete(self): # Cache keys can be deleted cache.set_many({'key1': 'spam', 'key2': 'eggs'}) self.assertEqual(cache.get('key1'), 'spam') - cache.delete('key1') + self.assertIs(cache.delete('key1'), True) self.assertIsNone(cache.get('key1')) self.assertEqual(cache.get('key2'), 'eggs') + def test_delete_nonexistent(self): + self.assertIs(cache.delete('nonexistent_key'), False) + def test_has_key(self): # The cache can be inspected for cache keys cache.set('hello1', 'goodbye1') - self.assertTrue(cache.has_key('hello1')) - self.assertFalse(cache.has_key('goodbye1')) + self.assertIs(cache.has_key('hello1'), True) + self.assertIs(cache.has_key('goodbye1'), False) cache.set('no_expiry', 'here', None) - self.assertTrue(cache.has_key('no_expiry')) + self.assertIs(cache.has_key('no_expiry'), True) + cache.set('null', None) + self.assertIs( + cache.has_key('null'), + True if self.supports_get_with_default else False, + ) def test_in(self): # The in operator can be used to inspect cache contents cache.set('hello2', 'goodbye2') self.assertIn('hello2', cache) self.assertNotIn('goodbye2', cache) + cache.set('null', None) + if self.supports_get_with_default: + self.assertIn('null', cache) + else: + self.assertNotIn('null', cache) def test_incr(self): # Cache values can be incremented @@ -213,6 +246,9 @@ def test_incr(self): self.assertEqual(cache.incr('answer', -10), 42) with self.assertRaises(ValueError): cache.incr('does_not_exist') + cache.set('null', None) + with self.assertRaises(self.incr_decr_type_error): + cache.incr('null') def test_decr(self): # Cache values can be decremented @@ -224,6 +260,9 @@ def test_decr(self): self.assertEqual(cache.decr('answer', -10), 42) with self.assertRaises(ValueError): cache.decr('does_not_exist') + cache.set('null', None) + with self.assertRaises(self.incr_decr_type_error): + cache.decr('null') def test_close(self): self.assertTrue(hasattr(cache, 'close')) @@ -295,24 +334,23 @@ def test_expiration(self): time.sleep(2) self.assertIsNone(cache.get('expire1')) - cache.add('expire2', 'newvalue') + self.assertIs(cache.add('expire2', 'newvalue'), True) self.assertEqual(cache.get('expire2'), 'newvalue') - self.assertFalse(cache.has_key('expire3')) + self.assertIs(cache.has_key('expire3'), False) def test_touch(self): # cache.touch() updates the timeout. cache.set('expire1', 'very quickly', timeout=1) self.assertIs(cache.touch('expire1', timeout=4), True) time.sleep(2) - self.assertTrue(cache.has_key('expire1')) + self.assertIs(cache.has_key('expire1'), True) time.sleep(3) - self.assertFalse(cache.has_key('expire1')) - + self.assertIs(cache.has_key('expire1'), False) # cache.touch() works without the timeout argument. cache.set('expire1', 'very quickly', timeout=1) self.assertIs(cache.touch('expire1'), True) time.sleep(2) - self.assertTrue(cache.has_key('expire1')) + self.assertIs(cache.has_key('expire1'), True) self.assertIs(cache.touch('nonexistent'), False) @@ -333,13 +371,13 @@ def test_unicode(self): # Test `add` for (key, value) in stuff.items(): with self.subTest(key=key): - cache.delete(key) - cache.add(key, value) + self.assertIs(cache.delete(key), True) + self.assertIs(cache.add(key, value), True) self.assertEqual(cache.get(key), value) # Test `set_many` for (key, value) in stuff.items(): - cache.delete(key) + self.assertIs(cache.delete(key), True) cache.set_many(stuff) for (key, value) in stuff.items(): with self.subTest(key=key): @@ -359,7 +397,7 @@ def test_binary_string(self): self.assertEqual(value, decompress(compressed_result).decode()) # Test add - cache.add('binary1-add', compressed_value) + self.assertIs(cache.add('binary1-add', compressed_value), True) compressed_result = cache.get('binary1-add') self.assertEqual(compressed_value, compressed_result) self.assertEqual(value, decompress(compressed_result).decode()) @@ -405,14 +443,14 @@ def test_clear(self): def test_long_timeout(self): """ - Followe memcached's convention where a timeout greater than 30 days is + Follow memcached's convention where a timeout greater than 30 days is treated as an absolute expiration timestamp instead of a relative offset (#12399). """ cache.set('key1', 'eggs', 60 * 60 * 24 * 30 + 1) # 30 days + 1 second self.assertEqual(cache.get('key1'), 'eggs') - cache.add('key2', 'ham', 60 * 60 * 24 * 30 + 1) + self.assertIs(cache.add('key2', 'ham', 60 * 60 * 24 * 30 + 1), True) self.assertEqual(cache.get('key2'), 'ham') cache.set_many( @@ -429,10 +467,9 @@ def test_forever_timeout(self): cache.set('key1', 'eggs', None) self.assertEqual(cache.get('key1'), 'eggs') - cache.add('key2', 'ham', None) + self.assertIs(cache.add('key2', 'ham', None), True) self.assertEqual(cache.get('key2'), 'ham') - added = cache.add('key1', 'new eggs', None) - self.assertIs(added, False) + self.assertIs(cache.add('key1', 'new eggs', None), False) self.assertEqual(cache.get('key1'), 'eggs') cache.set_many({'key3': 'sausage', 'key4': 'lobster bisque'}, None) @@ -440,7 +477,7 @@ def test_forever_timeout(self): self.assertEqual(cache.get('key4'), 'lobster bisque') cache.set('key5', 'belgian fries', timeout=1) - cache.touch('key5', timeout=None) + self.assertIs(cache.touch('key5', timeout=None), True) time.sleep(2) self.assertEqual(cache.get('key5'), 'belgian fries') @@ -451,7 +488,7 @@ def test_zero_timeout(self): cache.set('key1', 'eggs', 0) self.assertIsNone(cache.get('key1')) - cache.add('key2', 'ham', 0) + self.assertIs(cache.add('key2', 'ham', 0), True) self.assertIsNone(cache.get('key2')) cache.set_many({'key3': 'sausage', 'key4': 'lobster bisque'}, 0) @@ -459,7 +496,7 @@ def test_zero_timeout(self): self.assertIsNone(cache.get('key4')) cache.set('key5', 'belgian fries', timeout=5) - cache.touch('key5', timeout=0) + self.assertIs(cache.touch('key5', timeout=0), True) self.assertIsNone(cache.get('key5')) def test_float_timeout(self): @@ -467,7 +504,12 @@ def test_float_timeout(self): cache.set('key1', 'spam', 100.2) self.assertEqual(cache.get('key1'), 'spam') - def _perform_cull_test(self, cull_cache, initial_count, final_count): + def _perform_cull_test(self, cull_cache_name, initial_count, final_count): + try: + cull_cache = caches[cull_cache_name] + except InvalidCacheBackendError: + self.skipTest("Culling isn't implemented.") + # Create initial cache key entries. This will overflow the cache, # causing a cull. for i in range(1, initial_count): @@ -480,10 +522,24 @@ def _perform_cull_test(self, cull_cache, initial_count, final_count): self.assertEqual(count, final_count) def test_cull(self): - self._perform_cull_test(caches['cull'], 50, 29) + self._perform_cull_test('cull', 50, 29) def test_zero_cull(self): - self._perform_cull_test(caches['zero_cull'], 50, 19) + self._perform_cull_test('zero_cull', 50, 19) + + def test_cull_delete_when_store_empty(self): + try: + cull_cache = caches['cull'] + except InvalidCacheBackendError: + self.skipTest("Culling isn't implemented.") + old_max_entries = cull_cache._max_entries + # Force _cull to delete on first cached record. + cull_cache._max_entries = -1 + try: + cull_cache.set('force_cull_delete', 'value', 1000) + self.assertIs(cull_cache.has_key('force_cull_delete'), True) + finally: + cull_cache._max_entries = old_max_entries def _perform_invalid_key_test(self, key, expected_warning): """ @@ -500,10 +556,24 @@ def func(key, *args): old_func = cache.key_func cache.key_func = func + tests = [ + ('add', [key, 1]), + ('get', [key]), + ('set', [key, 1]), + ('incr', [key]), + ('decr', [key]), + ('touch', [key]), + ('delete', [key]), + ('get_many', [[key, 'b']]), + ('set_many', [{key: 1, 'b': 2}]), + ('delete_many', [{key: 1, 'b': 2}]), + ] try: - with self.assertWarns(CacheKeyWarning) as cm: - cache.set(key, 'value') - self.assertEqual(str(cm.warning), expected_warning) + for operation, args in tests: + with self.subTest(operation=operation): + with self.assertWarns(CacheKeyWarning) as cm: + getattr(cache, operation)(*args) + self.assertEqual(str(cm.warning), expected_warning) finally: cache.key_func = old_func @@ -567,41 +637,41 @@ def test_cache_versioning_get_set(self): def test_cache_versioning_add(self): # add, default version = 1, but manually override version = 2 - cache.add('answer1', 42, version=2) + self.assertIs(cache.add('answer1', 42, version=2), True) self.assertIsNone(cache.get('answer1', version=1)) self.assertEqual(cache.get('answer1', version=2), 42) - cache.add('answer1', 37, version=2) + self.assertIs(cache.add('answer1', 37, version=2), False) self.assertIsNone(cache.get('answer1', version=1)) self.assertEqual(cache.get('answer1', version=2), 42) - cache.add('answer1', 37, version=1) + self.assertIs(cache.add('answer1', 37, version=1), True) self.assertEqual(cache.get('answer1', version=1), 37) self.assertEqual(cache.get('answer1', version=2), 42) # v2 add, using default version = 2 - caches['v2'].add('answer2', 42) + self.assertIs(caches['v2'].add('answer2', 42), True) self.assertIsNone(cache.get('answer2', version=1)) self.assertEqual(cache.get('answer2', version=2), 42) - caches['v2'].add('answer2', 37) + self.assertIs(caches['v2'].add('answer2', 37), False) self.assertIsNone(cache.get('answer2', version=1)) self.assertEqual(cache.get('answer2', version=2), 42) - caches['v2'].add('answer2', 37, version=1) + self.assertIs(caches['v2'].add('answer2', 37, version=1), True) self.assertEqual(cache.get('answer2', version=1), 37) self.assertEqual(cache.get('answer2', version=2), 42) # v2 add, default version = 2, but manually override version = 1 - caches['v2'].add('answer3', 42, version=1) + self.assertIs(caches['v2'].add('answer3', 42, version=1), True) self.assertEqual(cache.get('answer3', version=1), 42) self.assertIsNone(cache.get('answer3', version=2)) - caches['v2'].add('answer3', 37, version=1) + self.assertIs(caches['v2'].add('answer3', 37, version=1), False) self.assertEqual(cache.get('answer3', version=1), 42) self.assertIsNone(cache.get('answer3', version=2)) - caches['v2'].add('answer3', 37) + self.assertIs(caches['v2'].add('answer3', 37), True) self.assertEqual(cache.get('answer3', version=1), 42) self.assertEqual(cache.get('answer3', version=2), 37) @@ -609,73 +679,73 @@ def test_cache_versioning_has_key(self): cache.set('answer1', 42) # has_key - self.assertTrue(cache.has_key('answer1')) - self.assertTrue(cache.has_key('answer1', version=1)) - self.assertFalse(cache.has_key('answer1', version=2)) + self.assertIs(cache.has_key('answer1'), True) + self.assertIs(cache.has_key('answer1', version=1), True) + self.assertIs(cache.has_key('answer1', version=2), False) - self.assertFalse(caches['v2'].has_key('answer1')) - self.assertTrue(caches['v2'].has_key('answer1', version=1)) - self.assertFalse(caches['v2'].has_key('answer1', version=2)) + self.assertIs(caches['v2'].has_key('answer1'), False) + self.assertIs(caches['v2'].has_key('answer1', version=1), True) + self.assertIs(caches['v2'].has_key('answer1', version=2), False) def test_cache_versioning_delete(self): cache.set('answer1', 37, version=1) cache.set('answer1', 42, version=2) - cache.delete('answer1') + self.assertIs(cache.delete('answer1'), True) self.assertIsNone(cache.get('answer1', version=1)) self.assertEqual(cache.get('answer1', version=2), 42) cache.set('answer2', 37, version=1) cache.set('answer2', 42, version=2) - cache.delete('answer2', version=2) + self.assertIs(cache.delete('answer2', version=2), True) self.assertEqual(cache.get('answer2', version=1), 37) self.assertIsNone(cache.get('answer2', version=2)) cache.set('answer3', 37, version=1) cache.set('answer3', 42, version=2) - caches['v2'].delete('answer3') + self.assertIs(caches['v2'].delete('answer3'), True) self.assertEqual(cache.get('answer3', version=1), 37) self.assertIsNone(cache.get('answer3', version=2)) cache.set('answer4', 37, version=1) cache.set('answer4', 42, version=2) - caches['v2'].delete('answer4', version=1) + self.assertIs(caches['v2'].delete('answer4', version=1), True) self.assertIsNone(cache.get('answer4', version=1)) self.assertEqual(cache.get('answer4', version=2), 42) def test_cache_versioning_incr_decr(self): cache.set('answer1', 37, version=1) cache.set('answer1', 42, version=2) - cache.incr('answer1') + self.assertEqual(cache.incr('answer1'), 38) self.assertEqual(cache.get('answer1', version=1), 38) self.assertEqual(cache.get('answer1', version=2), 42) - cache.decr('answer1') + self.assertEqual(cache.decr('answer1'), 37) self.assertEqual(cache.get('answer1', version=1), 37) self.assertEqual(cache.get('answer1', version=2), 42) cache.set('answer2', 37, version=1) cache.set('answer2', 42, version=2) - cache.incr('answer2', version=2) + self.assertEqual(cache.incr('answer2', version=2), 43) self.assertEqual(cache.get('answer2', version=1), 37) self.assertEqual(cache.get('answer2', version=2), 43) - cache.decr('answer2', version=2) + self.assertEqual(cache.decr('answer2', version=2), 42) self.assertEqual(cache.get('answer2', version=1), 37) self.assertEqual(cache.get('answer2', version=2), 42) cache.set('answer3', 37, version=1) cache.set('answer3', 42, version=2) - caches['v2'].incr('answer3') + self.assertEqual(caches['v2'].incr('answer3'), 43) self.assertEqual(cache.get('answer3', version=1), 37) self.assertEqual(cache.get('answer3', version=2), 43) - caches['v2'].decr('answer3') + self.assertEqual(caches['v2'].decr('answer3'), 42) self.assertEqual(cache.get('answer3', version=1), 37) self.assertEqual(cache.get('answer3', version=2), 42) cache.set('answer4', 37, version=1) cache.set('answer4', 42, version=2) - caches['v2'].incr('answer4', version=1) + self.assertEqual(caches['v2'].incr('answer4', version=1), 38) self.assertEqual(cache.get('answer4', version=1), 38) self.assertEqual(cache.get('answer4', version=2), 42) - caches['v2'].decr('answer4', version=1) + self.assertEqual(caches['v2'].decr('answer4', version=1), 37) self.assertEqual(cache.get('answer4', version=1), 37) self.assertEqual(cache.get('answer4', version=2), 42) @@ -790,6 +860,13 @@ def test_incr_version(self): with self.assertRaises(ValueError): cache.incr_version('does_not_exist') + cache.set('null', None) + if self.supports_get_with_default: + self.assertEqual(cache.incr_version('null'), 2) + else: + with self.assertRaises(self.incr_decr_type_error): + cache.incr_version('null') + def test_decr_version(self): cache.set('answer', 42, version=2) self.assertIsNone(cache.get('answer')) @@ -814,6 +891,13 @@ def test_decr_version(self): with self.assertRaises(ValueError): cache.decr_version('does_not_exist', version=2) + cache.set('null', None, version=2) + if self.supports_get_with_default: + self.assertEqual(cache.decr_version('null', version=2), 1) + else: + with self.assertRaises(self.incr_decr_type_error): + cache.decr_version('null', version=2) + def test_custom_key_func(self): # Two caches with different key functions aren't visible to each other cache.set('answer1', 42) @@ -827,30 +911,33 @@ def test_custom_key_func(self): self.assertEqual(caches['custom_key2'].get('answer2'), 42) def test_cache_write_unpicklable_object(self): - update_middleware = UpdateCacheMiddleware() - update_middleware.cache = cache - - fetch_middleware = FetchFromCacheMiddleware() + fetch_middleware = FetchFromCacheMiddleware(empty_response) fetch_middleware.cache = cache request = self.factory.get('/cache/test') request._cache_update_cache = True - get_cache_data = FetchFromCacheMiddleware().process_request(request) + get_cache_data = FetchFromCacheMiddleware( + empty_response + ).process_request(request) self.assertIsNone(get_cache_data) - response = HttpResponse() content = 'Testing cookie serialization.' - response.content = content - response.set_cookie('foo', 'bar') - update_middleware.process_response(request, response) + def get_response(req): + response = HttpResponse(content) + response.set_cookie('foo', 'bar') + return response + + update_middleware = UpdateCacheMiddleware(get_response) + update_middleware.cache = cache + response = update_middleware(request) get_cache_data = fetch_middleware.process_request(request) self.assertIsNotNone(get_cache_data) self.assertEqual(get_cache_data.content, content.encode()) self.assertEqual(get_cache_data.cookies, response.cookies) - update_middleware.process_response(request, get_cache_data) + UpdateCacheMiddleware(lambda req: get_cache_data)(request) get_cache_data = fetch_middleware.process_request(request) self.assertIsNotNone(get_cache_data) self.assertEqual(get_cache_data.content, content.encode()) @@ -869,7 +956,12 @@ def test_get_or_set(self): self.assertIsNone(cache.get('projector')) self.assertEqual(cache.get_or_set('projector', 42), 42) self.assertEqual(cache.get('projector'), 42) - self.assertEqual(cache.get_or_set('null', None), None) + self.assertIsNone(cache.get_or_set('null', None)) + if self.supports_get_with_default: + # Previous get_or_set() stores None in the cache. + self.assertIsNone(cache.get('null', 'default')) + else: + self.assertEqual(cache.get('null', 'default'), 'default') def test_get_or_set_callable(self): def my_callable(): @@ -878,14 +970,16 @@ def my_callable(): self.assertEqual(cache.get_or_set('mykey', my_callable), 'value') self.assertEqual(cache.get_or_set('mykey', my_callable()), 'value') - def test_get_or_set_callable_returning_none(self): - self.assertIsNone(cache.get_or_set('mykey', lambda: None)) - # Previous get_or_set() doesn't store None in the cache. - self.assertEqual(cache.get('mykey', 'default'), 'default') + self.assertIsNone(cache.get_or_set('null', lambda: None)) + if self.supports_get_with_default: + # Previous get_or_set() stores None in the cache. + self.assertIsNone(cache.get('null', 'default')) + else: + self.assertEqual(cache.get('null', 'default'), 'default') def test_get_or_set_version(self): msg = "get_or_set() missing 1 required positional argument: 'default'" - cache.get_or_set('brian', 1979, version=2) + self.assertEqual(cache.get_or_set('brian', 1979, version=2), 1979) with self.assertRaisesMessage(TypeError, msg): cache.get_or_set('brian') with self.assertRaisesMessage(TypeError, msg): @@ -949,6 +1043,11 @@ def test_ignores_non_cache_files(self): ) os.remove(fname) + def test_creates_cache_dir_if_nonexistent(self): + os.rmdir(self.dirname) + cache.set('foo', 'bar') + self.assertTrue(os.path.exists(self.dirname)) + def test_clear_does_not_remove_cache_dir(self): cache.clear() self.assertTrue( @@ -1026,19 +1125,6 @@ def test_pop(self): ) self.assertEqual(cache.pop(4, retry=False), 4) - def test_pickle(self): - letters = 'abcde' - cache.clear() - - for num, val in enumerate(letters): - cache.set(val, num) - - data = pickle.dumps(cache) - other = pickle.loads(data) - - for key in letters: - self.assertEqual(other.get(key), cache.get(key)) - def test_cache(self): subcache = cache.cache('test') directory = os.path.join(cache.directory, 'cache', 'test') From f3836f9f1938ede1eaf32d474be0311d0c7a3183 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Thu, 30 Dec 2021 17:35:55 -0800 Subject: [PATCH 089/126] Fix DjangoCache.delete to return True/False --- diskcache/djangocache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/diskcache/djangocache.py b/diskcache/djangocache.py index 347a613..8bf85ce 100644 --- a/diskcache/djangocache.py +++ b/diskcache/djangocache.py @@ -220,7 +220,7 @@ def delete(self, key, version=None, retry=True): """ # pylint: disable=arguments-differ key = self.make_key(key, version=version) - self._cache.delete(key, retry) + return self._cache.delete(key, retry) def incr(self, key, delta=1, version=None, default=None, retry=True): """Increment value by delta for item with key. From 8da6634877178ff3704d8468aa5a95b9baf4f9ac Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Thu, 30 Dec 2021 17:37:38 -0800 Subject: [PATCH 090/126] Bump Django testing to 3.2 --- requirements.txt | 2 +- tox.ini | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index 2ab91c7..efb2160 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ -e . blue coverage -django==2.2.* +django==3.2.* django_redis doc8 flake8 diff --git a/tox.ini b/tox.ini index 104a084..650c8f0 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,7 @@ skip_missing_interpreters=True [testenv] commands=pytest deps= - django==2.2.* + django==3.2.* pytest pytest-cov pytest-django @@ -31,7 +31,7 @@ allowlist_externals=make changedir=docs commands=make html deps= - django==2.2.* + django==3.2.* sphinx [testenv:flake8] @@ -53,7 +53,7 @@ deps=mypy [testenv:pylint] commands=pylint {toxinidir}/diskcache deps= - django==2.2.* + django==3.2.* pylint [testenv:rstcheck] From 221f3d38cea69e33ba9a83cde0e2e202f3d3d70e Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Thu, 30 Dec 2021 17:45:26 -0800 Subject: [PATCH 091/126] Remove unused imports --- tests/test_djangocache.py | 38 ++------------------------------------ 1 file changed, 2 insertions(+), 36 deletions(-) diff --git a/tests/test_djangocache.py b/tests/test_djangocache.py index 36e1b01..6fae5be 100644 --- a/tests/test_djangocache.py +++ b/tests/test_djangocache.py @@ -3,64 +3,30 @@ # Unit tests for cache framework # Uses whatever cache backend is set in the test settings file. -import copy -import io import os import pickle -import re import shutil -import sys import tempfile -import threading import time -import unittest -import warnings -from pathlib import Path -from unittest import mock, skipIf +from unittest import mock from django.conf import settings -from django.core import management, signals from django.core.cache import ( - DEFAULT_CACHE_ALIAS, CacheKeyWarning, - InvalidCacheKey, cache, caches, ) -from django.core.cache.utils import make_template_fragment_key -from django.db import close_old_connections, connection, connections -from django.http import ( - HttpRequest, - HttpResponse, - HttpResponseNotModified, - StreamingHttpResponse, -) +from django.http import HttpResponse from django.middleware.cache import ( - CacheMiddleware, FetchFromCacheMiddleware, UpdateCacheMiddleware, ) -from django.middleware.csrf import CsrfViewMiddleware -from django.template import engines -from django.template.context_processors import csrf -from django.template.response import TemplateResponse from django.test import ( RequestFactory, - SimpleTestCase, TestCase, - TransactionTestCase, override_settings, ) from django.test.signals import setting_changed -from django.utils import timezone, translation -from django.utils.cache import ( - get_cache_key, - learn_cache_key, - patch_cache_control, - patch_vary_headers, -) -from django.utils.encoding import force_text -from django.views.decorators.cache import cache_page ################################################################################ # Setup Django for models import. From 5ac77969c88df42eece83be2e4ce85d2e277109d Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Thu, 30 Dec 2021 17:48:08 -0800 Subject: [PATCH 092/126] Run isort --- tests/test_djangocache.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/tests/test_djangocache.py b/tests/test_djangocache.py index 6fae5be..5f83b81 100644 --- a/tests/test_djangocache.py +++ b/tests/test_djangocache.py @@ -11,21 +11,13 @@ from unittest import mock from django.conf import settings -from django.core.cache import ( - CacheKeyWarning, - cache, - caches, -) +from django.core.cache import CacheKeyWarning, cache, caches from django.http import HttpResponse from django.middleware.cache import ( FetchFromCacheMiddleware, UpdateCacheMiddleware, ) -from django.test import ( - RequestFactory, - TestCase, - override_settings, -) +from django.test import RequestFactory, TestCase, override_settings from django.test.signals import setting_changed ################################################################################ From 1cb1425b1ba24f26fb1e37349c4c2658c2a46d8f Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Thu, 30 Dec 2021 17:58:24 -0800 Subject: [PATCH 093/126] Bump version to 5.4.0 --- diskcache/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/diskcache/__init__.py b/diskcache/__init__.py index c361ca9..2355128 100644 --- a/diskcache/__init__.py +++ b/diskcache/__init__.py @@ -61,8 +61,8 @@ pass __title__ = 'diskcache' -__version__ = '5.3.0' -__build__ = 0x050300 +__version__ = '5.4.0' +__build__ = 0x050400 __author__ = 'Grant Jenks' __license__ = 'Apache 2.0' __copyright__ = 'Copyright 2016-2022 Grant Jenks' From c9844bba9de039ab64858073d8864f7e40172c9e Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Sun, 20 Feb 2022 12:47:41 -0800 Subject: [PATCH 094/126] Put commands above deps for doc8 testenv --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 650c8f0..65da17c 100644 --- a/tox.ini +++ b/tox.ini @@ -23,8 +23,8 @@ commands=blue --check {toxinidir}/setup.py {toxinidir}/diskcache {toxinidir}/tes deps=blue [testenv:doc8] -deps=doc8 commands=doc8 docs --ignore-path docs/_build +deps=doc8 [testenv:docs] allowlist_externals=make From 2d2cc8b39db3e0c0785dd304bde7902219d85ffe Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Sun, 20 Feb 2022 13:42:14 -0800 Subject: [PATCH 095/126] Update rsync command for uploading docs --- tox.ini | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 65da17c..36ed2b8 100644 --- a/tox.ini +++ b/tox.ini @@ -64,8 +64,9 @@ deps=rstcheck allowlist_externals=rsync changedir=docs commands= - rsync -azP --stats --delete _build/html/ \ - grantjenks.com:/srv/www/www.grantjenks.com/public/docs/diskcache/ + rsync --rsync-path 'sudo -u herokuish rsync' -azP --stats --delete \ + _build/html/ \ + grantjenks:/srv/www/grantjenks.com/public/docs/diskcache/ [isort] multi_line_output = 3 From c1774469b8d4c4906fe24f7b5afd637795af48d9 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 10 Apr 2023 22:12:28 -0700 Subject: [PATCH 096/126] Remove unused import --- setup.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/setup.py b/setup.py index b49d9c8..841dfb9 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,3 @@ -from io import open - from setuptools import setup from setuptools.command.test import test as TestCommand From f3fcdffee88af7923740b9864d533524a79d1222 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 10 Apr 2023 22:27:48 -0700 Subject: [PATCH 097/126] Update Cache(...) params when allocating --- diskcache/fanout.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/diskcache/fanout.py b/diskcache/fanout.py index dc5240c..8fe51d9 100644 --- a/diskcache/fanout.py +++ b/diskcache/fanout.py @@ -573,9 +573,11 @@ def reset(self, key, value=ENOVAL): break return result - def cache(self, name): + def cache(self, name, timeout=60, disk=None, **settings): """Return Cache with given `name` in subdirectory. + If disk is none (default), uses the fanout cache disk. + >>> fanout_cache = FanoutCache() >>> cache = fanout_cache.cache('test') >>> cache.set('abc', 123) @@ -588,6 +590,9 @@ def cache(self, name): True :param str name: subdirectory name for Cache + :param float timeout: SQLite connection timeout + :param disk: Disk type or subclass for serialization + :param settings: any of DEFAULT_SETTINGS :return: Cache with given name """ @@ -598,7 +603,12 @@ def cache(self, name): except KeyError: parts = name.split('/') directory = op.join(self._directory, 'cache', *parts) - temp = Cache(directory=directory, disk=self._disk) + temp = Cache( + directory=directory, + timeout=timeout, + disk=self._disk if disk is None else Disk, + **settings, + ) _caches[name] = temp return temp @@ -626,7 +636,11 @@ def deque(self, name): except KeyError: parts = name.split('/') directory = op.join(self._directory, 'deque', *parts) - cache = Cache(directory=directory, disk=self._disk) + cache = Cache( + directory=directory, + disk=self._disk, + eviction_policy='none', + ) deque = Deque.fromcache(cache) _deques[name] = deque return deque @@ -658,7 +672,11 @@ def index(self, name): except KeyError: parts = name.split('/') directory = op.join(self._directory, 'index', *parts) - cache = Cache(directory=directory, disk=self._disk) + cache = Cache( + directory=directory, + disk=self._disk, + eviction_policy='none', + ) index = Index.fromcache(cache) _indexes[name] = index return index From ee7a248e5c09e6fb9145b2e4a1777a345114b71d Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 10 Apr 2023 22:37:39 -0700 Subject: [PATCH 098/126] Add docs about the eviction policy to recipes --- diskcache/recipes.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/diskcache/recipes.py b/diskcache/recipes.py index b5af6dd..babb68f 100644 --- a/diskcache/recipes.py +++ b/diskcache/recipes.py @@ -17,6 +17,9 @@ class Averager: Sometimes known as "online statistics," the running average maintains the total and count. The average can then be calculated at any time. + Assumes the key will not be evicted. Set the eviction policy to 'none' on + the cache to guarantee the key is not evicted. + >>> import diskcache >>> cache = diskcache.FanoutCache() >>> ave = Averager(cache, 'latency') @@ -65,6 +68,9 @@ def pop(self): class Lock: """Recipe for cross-process and cross-thread lock. + Assumes the key will not be evicted. Set the eviction policy to 'none' on + the cache to guarantee the key is not evicted. + >>> import diskcache >>> cache = diskcache.Cache() >>> lock = Lock(cache, 'report-123') @@ -113,6 +119,9 @@ def __exit__(self, *exc_info): class RLock: """Recipe for cross-process and cross-thread re-entrant lock. + Assumes the key will not be evicted. Set the eviction policy to 'none' on + the cache to guarantee the key is not evicted. + >>> import diskcache >>> cache = diskcache.Cache() >>> rlock = RLock(cache, 'user-123') @@ -181,6 +190,9 @@ def __exit__(self, *exc_info): class BoundedSemaphore: """Recipe for cross-process and cross-thread bounded semaphore. + Assumes the key will not be evicted. Set the eviction policy to 'none' on + the cache to guarantee the key is not evicted. + >>> import diskcache >>> cache = diskcache.Cache() >>> semaphore = BoundedSemaphore(cache, 'max-cons', value=2) @@ -251,6 +263,9 @@ def throttle( ): """Decorator to throttle calls to function. + Assumes keys will not be evicted. Set the eviction policy to 'none' on the + cache to guarantee the keys are not evicted. + >>> import diskcache, time >>> cache = diskcache.Cache() >>> count = 0 @@ -305,6 +320,9 @@ def barrier(cache, lock_factory, name=None, expire=None, tag=None): Supports different kinds of locks: Lock, RLock, BoundedSemaphore. + Assumes keys will not be evicted. Set the eviction policy to 'none' on the + cache to guarantee the keys are not evicted. + >>> import diskcache, time >>> cache = diskcache.Cache() >>> @barrier(cache, Lock) From fb2fa2c401bb88ecbbc58009d02cbe65f2fc594a Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 10 Apr 2023 22:52:52 -0700 Subject: [PATCH 099/126] Test on Django 4.2 LTS --- tox.ini | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 36ed2b8..0ddcc8f 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,7 @@ skip_missing_interpreters=True [testenv] commands=pytest deps= - django==3.2.* + django==4.2.* pytest pytest-cov pytest-django @@ -31,7 +31,7 @@ allowlist_externals=make changedir=docs commands=make html deps= - django==3.2.* + django==4.2.* sphinx [testenv:flake8] @@ -53,7 +53,7 @@ deps=mypy [testenv:pylint] commands=pylint {toxinidir}/diskcache deps= - django==3.2.* + django==4.2.* pylint [testenv:rstcheck] From 0a9783353ff7dc9f874154ef98b23de27e4aba8d Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 10 Apr 2023 22:53:17 -0700 Subject: [PATCH 100/126] Update year to 2023 --- README.rst | 4 ++-- diskcache/__init__.py | 2 +- docs/conf.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index eb06a6a..04abdc0 100644 --- a/README.rst +++ b/README.rst @@ -4,7 +4,7 @@ DiskCache: Disk Backed Cache `DiskCache`_ is an Apache2 licensed disk and file backed cache library, written in pure-Python, and compatible with Django. -The cloud-based computing of 2021 puts a premium on memory. Gigabytes of empty +The cloud-based computing of 2023 puts a premium on memory. Gigabytes of empty space is left on disks as processes vie for memory. Among these processes is Memcached (and sometimes Redis) which is used as a cache. Wouldn't it be nice to leverage empty disk space for caching? @@ -387,7 +387,7 @@ Reference License ------- -Copyright 2016-2022 Grant Jenks +Copyright 2016-2023 Grant Jenks Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the diff --git a/diskcache/__init__.py b/diskcache/__init__.py index 2355128..f7aa771 100644 --- a/diskcache/__init__.py +++ b/diskcache/__init__.py @@ -65,4 +65,4 @@ __build__ = 0x050400 __author__ = 'Grant Jenks' __license__ = 'Apache 2.0' -__copyright__ = 'Copyright 2016-2022 Grant Jenks' +__copyright__ = 'Copyright 2016-2023 Grant Jenks' diff --git a/docs/conf.py b/docs/conf.py index 92ce1b9..92bf3ec 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -20,7 +20,7 @@ # -- Project information ----------------------------------------------------- project = 'DiskCache' -copyright = '2022, Grant Jenks' +copyright = '2023, Grant Jenks' author = 'Grant Jenks' # The full version, including alpha/beta/rc tags From 712cc1827b29fb8f6a76803d32f87a5fb57f3c6e Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 10 Apr 2023 22:56:16 -0700 Subject: [PATCH 101/126] Bump python testing to 3.11 --- .github/workflows/integration.yml | 4 ++-- .github/workflows/release.yml | 2 +- tox.ini | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 07a5650..2d83ad3 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -16,7 +16,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: '3.10' + python-version: '3.11' - name: Install dependencies run: | pip install --upgrade pip @@ -32,7 +32,7 @@ jobs: max-parallel: 8 matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: [3.6, 3.7, 3.8, 3.9, '3.10'] + python-version: [3.7, 3.8, 3.9, '3.10', 3.11] steps: - name: Set up Python ${{ matrix.python-version }} x64 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1f89c14..676593c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -21,7 +21,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: '3.10' + python-version: '3.11' - name: Install dependencies run: | diff --git a/tox.ini b/tox.ini index 0ddcc8f..3735ebc 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist=bluecheck,doc8,docs,isortcheck,flake8,mypy,pylint,rstcheck,py36,py37,py38,py39 +envlist=bluecheck,doc8,docs,isortcheck,flake8,mypy,pylint,rstcheck,py37,py38,py39,py310,py311 skip_missing_interpreters=True [testenv] From d7ae0990b240e8ea22c1a4060f58900edd339a18 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 10 Apr 2023 23:07:29 -0700 Subject: [PATCH 102/126] i blue it --- diskcache/core.py | 10 ++--- diskcache/fanout.py | 2 +- tests/benchmark_glob.py | 2 +- tests/settings.py | 2 +- tests/stress_test_core.py | 30 ++++++------- tests/stress_test_fanout.py | 30 ++++++------- tests/test_core.py | 84 ++++++++++++++++++------------------- tests/test_djangocache.py | 2 +- tests/test_fanout.py | 16 +++---- 9 files changed, 85 insertions(+), 93 deletions(-) diff --git a/diskcache/core.py b/diskcache/core.py index fb343be..05a0854 100644 --- a/diskcache/core.py +++ b/diskcache/core.py @@ -49,14 +49,14 @@ def __repr__(self): 'statistics': 0, # False 'tag_index': 0, # False 'eviction_policy': 'least-recently-stored', - 'size_limit': 2 ** 30, # 1gb + 'size_limit': 2**30, # 1gb 'cull_limit': 10, 'sqlite_auto_vacuum': 1, # FULL - 'sqlite_cache_size': 2 ** 13, # 8,192 pages + 'sqlite_cache_size': 2**13, # 8,192 pages 'sqlite_journal_mode': 'wal', - 'sqlite_mmap_size': 2 ** 26, # 64mb + 'sqlite_mmap_size': 2**26, # 64mb 'sqlite_synchronous': 1, # NORMAL - 'disk_min_file_size': 2 ** 15, # 32kb + 'disk_min_file_size': 2**15, # 32kb 'disk_pickle_protocol': pickle.HIGHEST_PROTOCOL, } @@ -212,7 +212,7 @@ def store(self, value, read, key=UNKNOWN): size = op.getsize(full_path) return size, MODE_TEXT, filename, None elif read: - reader = ft.partial(value.read, 2 ** 22) + reader = ft.partial(value.read, 2**22) filename, full_path = self.filename(key, value) iterator = iter(reader, b'') size = self._write(full_path, iterator, 'xb') diff --git a/diskcache/fanout.py b/diskcache/fanout.py index 8fe51d9..5283490 100644 --- a/diskcache/fanout.py +++ b/diskcache/fanout.py @@ -45,7 +45,7 @@ def __init__( timeout=timeout, disk=disk, size_limit=size_limit, - **settings + **settings, ) for num in range(shards) ) diff --git a/tests/benchmark_glob.py b/tests/benchmark_glob.py index 7f0bf7c..7da5fd3 100644 --- a/tests/benchmark_glob.py +++ b/tests/benchmark_glob.py @@ -22,7 +22,7 @@ print(template % ('Count', 'Time')) print(' '.join(['=' * size] * len(cols))) -for count in [10 ** exp for exp in range(6)]: +for count in [10**exp for exp in range(6)]: for value in range(count): with open(op.join('tmp', '%s.tmp' % value), 'wb') as writer: pass diff --git a/tests/settings.py b/tests/settings.py index 1a2f569..04aee85 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -25,7 +25,7 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True -ALLOWED_HOSTS = [u'testserver'] +ALLOWED_HOSTS = ['testserver'] # Application definition diff --git a/tests/stress_test_core.py b/tests/stress_test_core.py index c30fa3f..2b2578b 100644 --- a/tests/stress_test_core.py +++ b/tests/stress_test_core.py @@ -33,16 +33,14 @@ def make_long(): def make_unicode(): word_size = random.randint(1, 26) - word = u''.join( - random.sample(u'abcdefghijklmnopqrstuvwxyz', word_size) - ) + word = ''.join(random.sample('abcdefghijklmnopqrstuvwxyz', word_size)) size = random.randint(1, int(200 / 13)) return word * size def make_bytes(): word_size = random.randint(1, 26) - word = u''.join( - random.sample(u'abcdefghijklmnopqrstuvwxyz', word_size) + word = ''.join( + random.sample('abcdefghijklmnopqrstuvwxyz', word_size) ).encode('utf-8') size = random.randint(1, int(200 / 13)) return word * size @@ -77,18 +75,16 @@ def make_long(): def make_unicode(): word_size = random.randint(1, 26) - word = u''.join( - random.sample(u'abcdefghijklmnopqrstuvwxyz', word_size) - ) - size = random.randint(1, int(2 ** 16 / 13)) + word = ''.join(random.sample('abcdefghijklmnopqrstuvwxyz', word_size)) + size = random.randint(1, int(2**16 / 13)) return word * size def make_bytes(): word_size = random.randint(1, 26) - word = u''.join( - random.sample(u'abcdefghijklmnopqrstuvwxyz', word_size) + word = ''.join( + random.sample('abcdefghijklmnopqrstuvwxyz', word_size) ).encode('utf-8') - size = random.randint(1, int(2 ** 16 / 13)) + size = random.randint(1, int(2**16 / 13)) return word * size def make_float(): @@ -233,7 +229,7 @@ def percentile(sequence, percent): def stress_test( create=True, delete=True, - eviction_policy=u'least-recently-stored', + eviction_policy='least-recently-stored', processes=1, threads=1, ): @@ -293,17 +289,17 @@ def stress_test( def stress_test_lru(): """Stress test least-recently-used eviction policy.""" - stress_test(eviction_policy=u'least-recently-used') + stress_test(eviction_policy='least-recently-used') def stress_test_lfu(): """Stress test least-frequently-used eviction policy.""" - stress_test(eviction_policy=u'least-frequently-used') + stress_test(eviction_policy='least-frequently-used') def stress_test_none(): """Stress test 'none' eviction policy.""" - stress_test(eviction_policy=u'none') + stress_test(eviction_policy='none') def stress_test_mp(): @@ -396,7 +392,7 @@ def stress_test_mp(): '-v', '--eviction-policy', type=str, - default=u'least-recently-stored', + default='least-recently-stored', ) args = parser.parse_args() diff --git a/tests/stress_test_fanout.py b/tests/stress_test_fanout.py index d3b67e3..e78dda5 100644 --- a/tests/stress_test_fanout.py +++ b/tests/stress_test_fanout.py @@ -32,16 +32,14 @@ def make_long(): def make_unicode(): word_size = random.randint(1, 26) - word = u''.join( - random.sample(u'abcdefghijklmnopqrstuvwxyz', word_size) - ) + word = ''.join(random.sample('abcdefghijklmnopqrstuvwxyz', word_size)) size = random.randint(1, int(200 / 13)) return word * size def make_bytes(): word_size = random.randint(1, 26) - word = u''.join( - random.sample(u'abcdefghijklmnopqrstuvwxyz', word_size) + word = ''.join( + random.sample('abcdefghijklmnopqrstuvwxyz', word_size) ).encode('utf-8') size = random.randint(1, int(200 / 13)) return word * size @@ -76,18 +74,16 @@ def make_long(): def make_unicode(): word_size = random.randint(1, 26) - word = u''.join( - random.sample(u'abcdefghijklmnopqrstuvwxyz', word_size) - ) - size = random.randint(1, int(2 ** 16 / 13)) + word = ''.join(random.sample('abcdefghijklmnopqrstuvwxyz', word_size)) + size = random.randint(1, int(2**16 / 13)) return word * size def make_bytes(): word_size = random.randint(1, 26) - word = u''.join( - random.sample(u'abcdefghijklmnopqrstuvwxyz', word_size) + word = ''.join( + random.sample('abcdefghijklmnopqrstuvwxyz', word_size) ).encode('utf-8') - size = random.randint(1, int(2 ** 16 / 13)) + size = random.randint(1, int(2**16 / 13)) return word * size def make_float(): @@ -224,7 +220,7 @@ def percentile(sequence, percent): def stress_test( create=True, delete=True, - eviction_policy=u'least-recently-stored', + eviction_policy='least-recently-stored', processes=1, threads=1, ): @@ -284,17 +280,17 @@ def stress_test( def stress_test_lru(): """Stress test least-recently-used eviction policy.""" - stress_test(eviction_policy=u'least-recently-used') + stress_test(eviction_policy='least-recently-used') def stress_test_lfu(): """Stress test least-frequently-used eviction policy.""" - stress_test(eviction_policy=u'least-frequently-used') + stress_test(eviction_policy='least-frequently-used') def stress_test_none(): """Stress test 'none' eviction policy.""" - stress_test(eviction_policy=u'none') + stress_test(eviction_policy='none') def stress_test_mp(): @@ -387,7 +383,7 @@ def stress_test_mp(): '-v', '--eviction-policy', type=str, - default=u'least-recently-stored', + default='least-recently-stored', ) args = parser.parse_args() diff --git a/tests/test_core.py b/tests/test_core.py index 55ca962..356d104 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -38,11 +38,11 @@ def test_init(cache): def test_init_disk(): - with dc.Cache(disk_pickle_protocol=1, disk_min_file_size=2 ** 20) as cache: + with dc.Cache(disk_pickle_protocol=1, disk_min_file_size=2**20) as cache: key = (None, 0, 'abc') cache[key] = 0 cache.check() - assert cache.disk_min_file_size == 2 ** 20 + assert cache.disk_min_file_size == 2**20 assert cache.disk_pickle_protocol == 1 shutil.rmtree(cache.directory, ignore_errors=True) @@ -59,15 +59,15 @@ def test_disk_reset(): assert cache._disk.min_file_size == 0 assert cache._disk.pickle_protocol == 0 - cache.reset('disk_min_file_size', 2 ** 10) + cache.reset('disk_min_file_size', 2**10) cache.reset('disk_pickle_protocol', 2) cache[1] = value cache.check() - assert cache.disk_min_file_size == 2 ** 10 + assert cache.disk_min_file_size == 2**10 assert cache.disk_pickle_protocol == 2 - assert cache._disk.min_file_size == 2 ** 10 + assert cache._disk.min_file_size == 2**10 assert cache._disk.pickle_protocol == 2 shutil.rmtree(cache.directory, ignore_errors=True) @@ -150,7 +150,7 @@ def test_pragma_error(cache): cursor.fetchall = fetchall fetchall.side_effect = [sqlite3.OperationalError] * 60000 - size = 2 ** 28 + size = 2**28 with mock.patch('time.sleep', lambda num: 0): with mock.patch.object(cache, '_local', local): @@ -177,15 +177,15 @@ def __getattr__(self, name): def test_getsetdel(cache): values = [ (None, False), - ((None,) * 2 ** 20, False), + ((None,) * 2**20, False), (1234, False), - (2 ** 512, False), + (2**512, False), (56.78, False), - (u'hello', False), - (u'hello' * 2 ** 20, False), + ('hello', False), + ('hello' * 2**20, False), (b'world', False), - (b'world' * 2 ** 20, False), - (io.BytesIO(b'world' * 2 ** 20), True), + (b'world' * 2**20, False), + (io.BytesIO(b'world' * 2**20), True), ] for key, (value, file_like) in enumerate(values): @@ -229,7 +229,7 @@ def test_get_keyerror4(cache): func = mock.Mock(side_effect=IOError(errno.ENOENT, '')) cache.reset('statistics', True) - cache[0] = b'abcd' * 2 ** 20 + cache[0] = b'abcd' * 2**20 with mock.patch('diskcache.core.open', func): with pytest.raises((IOError, KeyError, OSError)): @@ -237,7 +237,7 @@ def test_get_keyerror4(cache): def test_read(cache): - cache.set(0, b'abcd' * 2 ** 20) + cache.set(0, b'abcd' * 2**20) with cache.read(0) as reader: assert reader is not None @@ -249,7 +249,7 @@ def test_read_keyerror(cache): def test_set_twice(cache): - large_value = b'abcd' * 2 ** 20 + large_value = b'abcd' * 2**20 cache[0] = 0 cache[0] = 1 @@ -283,7 +283,7 @@ def test_set_timeout(cache): with pytest.raises(dc.Timeout): try: with mock.patch.object(cache, '_local', local): - cache.set('a', 'b' * 2 ** 20) + cache.set('a', 'b' * 2**20) finally: cache.check() @@ -299,11 +299,11 @@ def test_get(cache): assert cache.get(2, {}) == {} assert cache.get(0, expire_time=True, tag=True) == (None, None, None) - assert cache.set(0, 0, expire=None, tag=u'number') + assert cache.set(0, 0, expire=None, tag='number') assert cache.get(0, expire_time=True) == (0, None) - assert cache.get(0, tag=True) == (0, u'number') - assert cache.get(0, expire_time=True, tag=True) == (0, None, u'number') + assert cache.get(0, tag=True) == (0, 'number') + assert cache.get(0, expire_time=True, tag=True) == (0, None, 'number') def test_get_expired_fast_path(cache): @@ -359,8 +359,8 @@ def test_pop(cache): assert cache.set('delta', 210) assert cache.pop('delta', expire_time=True) == (210, None) - assert cache.set('epsilon', '0' * 2 ** 20) - assert cache.pop('epsilon') == '0' * 2 ** 20 + assert cache.set('epsilon', '0' * 2**20) + assert cache.pop('epsilon') == '0' * 2**20 def test_pop_ioerror(cache): @@ -426,11 +426,11 @@ def test_stats(cache): def test_path(cache): - cache[0] = u'abc' - large_value = b'abc' * 2 ** 20 + cache[0] = 'abc' + large_value = b'abc' * 2**20 cache[1] = large_value - assert cache.get(0, read=True) == u'abc' + assert cache.get(0, read=True) == 'abc' with cache.get(1, read=True) as reader: assert reader.name is not None @@ -465,7 +465,7 @@ def test_expire_rows(cache): def test_least_recently_stored(cache): - cache.reset('eviction_policy', u'least-recently-stored') + cache.reset('eviction_policy', 'least-recently-stored') cache.reset('size_limit', int(10.1e6)) cache.reset('cull_limit', 2) @@ -500,7 +500,7 @@ def test_least_recently_stored(cache): def test_least_recently_used(cache): - cache.reset('eviction_policy', u'least-recently-used') + cache.reset('eviction_policy', 'least-recently-used') cache.reset('size_limit', int(10.1e6)) cache.reset('cull_limit', 5) @@ -530,7 +530,7 @@ def test_least_recently_used(cache): def test_least_frequently_used(cache): - cache.reset('eviction_policy', u'least-frequently-used') + cache.reset('eviction_policy', 'least-frequently-used') cache.reset('size_limit', int(10.1e6)) cache.reset('cull_limit', 5) @@ -558,8 +558,8 @@ def test_least_frequently_used(cache): def test_check(cache): - blob = b'a' * 2 ** 20 - keys = (0, 1, 1234, 56.78, u'hello', b'world', None) + blob = b'a' * 2**20 + keys = (0, 1, 1234, 56.78, 'hello', b'world', None) for key in keys: cache[key] = blob @@ -662,12 +662,12 @@ def test_clear_timeout(cache): def test_tag(cache): - assert cache.set(0, None, tag=u'zero') + assert cache.set(0, None, tag='zero') assert cache.set(1, None, tag=1234) assert cache.set(2, None, tag=5.67) assert cache.set(3, None, tag=b'three') - assert cache.get(0, tag=True) == (None, u'zero') + assert cache.get(0, tag=True) == (None, 'zero') assert cache.get(1, tag=True) == (None, 1234) assert cache.get(2, tag=True) == (None, 5.67) assert cache.get(3, tag=True) == (None, b'three') @@ -675,11 +675,11 @@ def test_tag(cache): def test_with(cache): with dc.Cache(cache.directory) as tmp: - tmp[u'a'] = 0 - tmp[u'b'] = 1 + tmp['a'] = 0 + tmp['b'] = 1 - assert cache[u'a'] == 0 - assert cache[u'b'] == 1 + assert cache['a'] == 0 + assert cache['b'] == 1 def test_contains(cache): @@ -708,7 +708,7 @@ def test_add(cache): def test_add_large_value(cache): - value = b'abcd' * 2 ** 20 + value = b'abcd' * 2**20 assert cache.add(b'test-key', value) assert cache.get(b'test-key') == value assert not cache.add(b'test-key', value * 2) @@ -919,7 +919,7 @@ def test_push_peek_expire(cache): def test_push_pull_large_value(cache): - value = b'test' * (2 ** 20) + value = b'test' * (2**20) cache.push(value) assert cache.pull() == (500000000000000, value) assert len(cache) == 0 @@ -927,7 +927,7 @@ def test_push_pull_large_value(cache): def test_push_peek_large_value(cache): - value = b'test' * (2 ** 20) + value = b'test' * (2**20) cache.push(value) assert cache.peek() == (500000000000000, value) assert len(cache) == 1 @@ -1144,8 +1144,8 @@ def test_cull_timeout(cache): def test_key_roundtrip(cache): - key_part_0 = u'part0' - key_part_1 = u'part1' + key_part_0 = 'part0' + key_part_1 = 'part1' to_test = [ (key_part_0, key_part_1), [key_part_0, key_part_1], @@ -1354,7 +1354,7 @@ def foo(*args, **kwargs): def test_cleanup_dirs(cache): - value = b'\0' * 2 ** 20 + value = b'\0' * 2**20 start_count = len(os.listdir(cache.directory)) for i in range(10): cache[i] = value @@ -1370,7 +1370,7 @@ def test_disk_write_os_error(cache): func = mock.Mock(side_effect=[OSError] * 10) with mock.patch('diskcache.core.open', func): with pytest.raises(OSError): - cache[0] = '\0' * 2 ** 20 + cache[0] = '\0' * 2**20 def test_memoize_ignore(cache): diff --git a/tests/test_djangocache.py b/tests/test_djangocache.py index 5f83b81..b5cc2a8 100644 --- a/tests/test_djangocache.py +++ b/tests/test_djangocache.py @@ -1033,7 +1033,7 @@ def test_directory(self): self.assertTrue('tmp' in cache.directory) def test_read(self): - value = b'abcd' * 2 ** 20 + value = b'abcd' * 2**20 result = cache.set(b'test-key', value) self.assertTrue(result) diff --git a/tests/test_fanout.py b/tests/test_fanout.py index f212fac..deea03f 100644 --- a/tests/test_fanout.py +++ b/tests/test_fanout.py @@ -34,7 +34,7 @@ def test_init(cache): del default_settings['size_limit'] for key, value in default_settings.items(): assert getattr(cache, key) == value - assert cache.size_limit == 2 ** 27 + assert cache.size_limit == 2**27 cache.check() @@ -229,15 +229,15 @@ def test_incr_concurrent(): def test_getsetdel(cache): values = [ (None, False), - ((None,) * 2 ** 10, False), + ((None,) * 2**10, False), (1234, False), - (2 ** 512, False), + (2**512, False), (56.78, False), - (u'hello', False), - (u'hello' * 2 ** 10, False), + ('hello', False), + ('hello' * 2**10, False), (b'world', False), - (b'world' * 2 ** 10, False), - (io.BytesIO(b'world' * 2 ** 10), True), + (b'world' * 2**10, False), + (io.BytesIO(b'world' * 2**10), True), ] for key, (value, file_like) in enumerate(values): @@ -341,7 +341,7 @@ def test_tag_index(cache): def test_read(cache): - cache.set(0, b'abcd' * 2 ** 20) + cache.set(0, b'abcd' * 2**20) with cache.read(0) as reader: assert reader is not None From a53283d554fbe7bd678067f24d6f45d57de3f83b Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 10 Apr 2023 23:25:16 -0700 Subject: [PATCH 103/126] Update requirements --- requirements-dev.txt | 23 +++++++++++++++++++++++ requirements.txt | 22 ---------------------- 2 files changed, 23 insertions(+), 22 deletions(-) create mode 100644 requirements-dev.txt diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..6149361 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,23 @@ +-e . +blue +coverage +django==4.2.* +django_redis +doc8 +flake8 +ipython +jedi +pickleDB +pylibmc +pylint +pytest +pytest-cov +pytest-django +pytest-env +pytest-xdist +rstcheck +sphinx +sqlitedict +tox +twine +wheel diff --git a/requirements.txt b/requirements.txt index efb2160..d6e1198 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,23 +1 @@ -e . -blue -coverage -django==3.2.* -django_redis -doc8 -flake8 -ipython -jedi==0.17.* # Remove after IPython bug fixed. -pickleDB -pylibmc -pylint -pytest -pytest-cov -pytest-django -pytest-env -pytest-xdist -rstcheck -sphinx -sqlitedict -tox -twine -wheel From b22a7d58c3dbf3f71fa4f9156ccbd4892ebf3fe2 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 10 Apr 2023 23:25:35 -0700 Subject: [PATCH 104/126] Update pylint --- .pylintrc | 829 ++++++++++++++++++++++++---------------------- diskcache/core.py | 6 +- 2 files changed, 439 insertions(+), 396 deletions(-) diff --git a/.pylintrc b/.pylintrc index 6baa978..dc1490a 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,27 +1,77 @@ -[MASTER] +[MAIN] + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Clear in-memory caches upon conclusion of linting. Useful if running pylint +# in a server-like mode. +clear-cache-post-run=no + +# Load and enable all available extensions. Use --list-extensions to see a list +# all available extensions. +#enable-all-extensions= + +# In error mode, messages with a category besides ERROR or FATAL are +# suppressed, and no reports are done by default. Error mode is compatible with +# disabling specific errors. +#errors-only= + +# Always return a 0 (non-error) status code, even if lint errors are found. +# This is primarily useful in continuous integration scripts. +#exit-zero= # A comma-separated list of package or module names from where C extensions may # be loaded. Extensions are loading into the active Python interpreter and may # run arbitrary code. +extension-pkg-allow-list= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. (This is an alternative name to extension-pkg-allow-list +# for backward compatibility.) extension-pkg-whitelist= -# Specify a score threshold to be exceeded before program exits with error. -fail-under=10.0 +# Return non-zero exit code if any of these messages/categories are detected, +# even if score is above --fail-under value. Syntax same as enable. Messages +# specified are enabled, while categories only check already-enabled messages. +fail-on= -# Add files or directories to the blacklist. They should be base names, not -# paths. +# Specify a score threshold under which the program will exit with error. +fail-under=10 + +# Interpret the stdin as a python script, whose filename needs to be passed as +# the module_or_package argument. +#from-stdin= + +# Files or directories to be skipped. They should be base names, not paths. ignore=CVS -# Add files or directories matching the regex patterns to the blacklist. The -# regex matches against base names, not paths. -ignore-patterns= +# Add files or directories matching the regular expressions patterns to the +# ignore-list. The regex matches against paths and can be in Posix or Windows +# format. Because '\\' represents the directory delimiter on Windows systems, +# it can't be used as an escape character. +ignore-paths= + +# Files or directories matching the regular expression patterns are skipped. +# The regex matches against base names, not paths. The default value ignores +# Emacs file locks +ignore-patterns=^\.# + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis). It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= # Python code to execute, usually for sys.path manipulation such as # pygtk.require(). #init-hook= # Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the -# number of processors available to use. +# number of processors available to use, and will cap the count on Windows to +# avoid hangs. jobs=1 # Control the amount of potential inferred values when inferring a single @@ -36,6 +86,19 @@ load-plugins= # Pickle collected data for later comparisons. persistent=yes +# Minimum Python version to use for version dependent checks. Will default to +# the version used to run pylint. +py-version=3.11 + +# Discover python modules and packages in the file system subtree. +recursive=no + +# Add paths to the list of the source roots. Supports globbing patterns. The +# source root is an absolute path or a path relative to the current working +# directory used to determine a package namespace for modules located under the +# source root. +source-roots= + # When enabled, pylint would attempt to guess common misconfiguration and emit # user-friendly hints instead of false-positive error messages. suggestion-mode=yes @@ -44,321 +107,8 @@ suggestion-mode=yes # active Python interpreter and may run arbitrary code. unsafe-load-any-extension=no - -[MESSAGES CONTROL] - -# Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED. -confidence= - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once). You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use "--disable=all --enable=classes -# --disable=W". -disable=print-statement, - parameter-unpacking, - unpacking-in-except, - old-raise-syntax, - backtick, - long-suffix, - old-ne-operator, - old-octal-literal, - import-star-module-level, - non-ascii-bytes-literal, - raw-checker-failed, - bad-inline-option, - locally-disabled, - file-ignored, - suppressed-message, - useless-suppression, - deprecated-pragma, - use-symbolic-message-instead, - apply-builtin, - basestring-builtin, - buffer-builtin, - cmp-builtin, - coerce-builtin, - execfile-builtin, - file-builtin, - long-builtin, - raw_input-builtin, - reduce-builtin, - standarderror-builtin, - unicode-builtin, - xrange-builtin, - coerce-method, - delslice-method, - getslice-method, - setslice-method, - no-absolute-import, - old-division, - dict-iter-method, - dict-view-method, - next-method-called, - metaclass-assignment, - indexing-exception, - raising-string, - reload-builtin, - oct-method, - hex-method, - nonzero-method, - cmp-method, - input-builtin, - round-builtin, - intern-builtin, - unichr-builtin, - map-builtin-not-iterating, - zip-builtin-not-iterating, - range-builtin-not-iterating, - filter-builtin-not-iterating, - using-cmp-argument, - eq-without-hash, - div-method, - idiv-method, - rdiv-method, - exception-message-attribute, - invalid-str-codec, - sys-max-int, - bad-python3-import, - deprecated-string-function, - deprecated-str-translate-call, - deprecated-itertools-function, - deprecated-types-field, - next-method-defined, - dict-items-not-iterating, - dict-keys-not-iterating, - dict-values-not-iterating, - deprecated-operator-function, - deprecated-urllib-function, - xreadlines-attribute, - deprecated-sys-function, - exception-escape, - comprehension-escape, - no-member, - no-else-return, - duplicate-code, - inconsistent-return-statements, - consider-using-f-string, - -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time (only on the command line, not in the configuration file where -# it should appear only once). See also the "--disable" option for examples. -enable=c-extension-no-member - - -[REPORTS] - -# Python expression which should return a score less than or equal to 10. You -# have access to the variables 'error', 'warning', 'refactor', and 'convention' -# which contain the number of messages in each category, as well as 'statement' -# which is the total number of statements analyzed. This score is used by the -# global evaluation report (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) - -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details. -#msg-template= - -# Set the output format. Available formats are text, parseable, colorized, json -# and msvs (visual studio). You can also give a reporter class, e.g. -# mypackage.mymodule.MyReporterClass. -output-format=text - -# Tells whether to display a full report or only the messages. -reports=no - -# Activate the evaluation score. -score=yes - - -[REFACTORING] - -# Maximum number of nested blocks for function / method body -max-nested-blocks=5 - -# Complete name of functions that never returns. When checking for -# inconsistent-return-statements if a never returning function is called then -# it will be considered as an explicit return statement and no message will be -# printed. -never-returning-functions=sys.exit - - -[LOGGING] - -# The type of string formatting that logging methods do. `old` means using % -# formatting, `new` is for `{}` formatting. -logging-format-style=old - -# Logging modules to check that the string format arguments are in logging -# function parameter format. -logging-modules=logging - - -[SPELLING] - -# Limits count of emitted suggestions for spelling mistakes. -max-spelling-suggestions=4 - -# Spelling dictionary name. Available dictionaries: none. To make it work, -# install the python-enchant package. -spelling-dict= - -# List of comma separated words that should not be checked. -spelling-ignore-words= - -# A path to a file that contains the private dictionary; one word per line. -spelling-private-dict-file= - -# Tells whether to store unknown words to the private dictionary (see the -# --spelling-private-dict-file option) instead of raising a message. -spelling-store-unknown-words=no - - -[MISCELLANEOUS] - -# List of note tags to take in consideration, separated by a comma. -notes=FIXME, - XXX, - TODO - -# Regular expression of note tags to take in consideration. -#notes-rgx= - - -[TYPECHECK] - -# List of decorators that produce context managers, such as -# contextlib.contextmanager. Add to this list to register other decorators that -# produce valid context managers. -contextmanager-decorators=contextlib.contextmanager - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. -generated-members= - -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes - -# Tells whether to warn about missing members when the owner of the attribute -# is inferred to be None. -ignore-none=yes - -# This flag controls whether pylint should warn about no-member and similar -# checks whenever an opaque object is returned when inferring. The inference -# can return multiple potential results while evaluating a Python object, but -# some branches might not be evaluated, which results in partial inference. In -# that case, it might be useful to still emit no-member and other checks for -# the rest of the inferred objects. -ignore-on-opaque-inference=yes - -# List of class names for which member attributes should not be checked (useful -# for classes with dynamically set attributes). This supports the use of -# qualified names. -ignored-classes=optparse.Values,thread._local,_thread._local - -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis). It -# supports qualified module names, as well as Unix pattern matching. -ignored-modules= - -# Show a hint with possible names when a member name was not found. The aspect -# of finding the hint is based on edit distance. -missing-member-hint=yes - -# The minimum edit distance a name should have in order to be considered a -# similar match for a missing member name. -missing-member-hint-distance=1 - -# The total number of similar names that should be taken in consideration when -# showing a hint for a missing member. -missing-member-max-choices=1 - -# List of decorators that change the signature of a decorated function. -signature-mutators= - - -[VARIABLES] - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid defining new builtins when possible. -additional-builtins= - -# Tells whether unused global variables should be treated as a violation. -allow-global-unused-variables=yes - -# List of strings which can identify a callback function by name. A callback -# name must start or end with one of those strings. -callbacks=cb_, - _cb - -# A regular expression matching the name of dummy variables (i.e. expected to -# not be used). -dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ - -# Argument names that match this expression will be ignored. Default to name -# with leading underscore. -ignored-argument-names=_.*|^ignored_|^unused_ - -# Tells whether we should check for unused import in __init__ files. -init-import=no - -# List of qualified module names which can have objects that can redefine -# builtins. -redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io - - -[FORMAT] - -# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. -expected-line-ending-format= - -# Regexp for a line that is allowed to be longer than the limit. -ignore-long-lines=^\s*(# )??$ - -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren=4 - -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string=' ' - -# Maximum number of characters on a single line. -max-line-length=100 - -# Maximum number of lines in a module. -max-module-lines=3000 - -# Allow the body of a class to be on the same line as the declaration if body -# contains single statement. -single-line-class-stmt=no - -# Allow the body of an if to be on the same line as the test if there is no -# else. -single-line-if-stmt=no - - -[SIMILARITIES] - -# Ignore comments when computing similarities. -ignore-comments=yes - -# Ignore docstrings when computing similarities. -ignore-docstrings=yes - -# Ignore imports when computing similarities. -ignore-imports=yes - -# Minimum lines number of a similarity. -min-similarity-lines=4 +# In verbose mode, extra non-checker-related info will be displayed. +#verbose= [BASIC] @@ -367,13 +117,15 @@ min-similarity-lines=4 argument-naming-style=snake_case # Regular expression matching correct argument names. Overrides argument- -# naming-style. +# naming-style. If left empty, argument names will be checked with the set +# naming style. #argument-rgx= # Naming style matching correct attribute names. attr-naming-style=snake_case # Regular expression matching correct attribute names. Overrides attr-naming- +# style. If left empty, attribute names will be checked with the set naming # style. #attr-rgx= @@ -393,20 +145,30 @@ bad-names-rgxs= class-attribute-naming-style=any # Regular expression matching correct class attribute names. Overrides class- -# attribute-naming-style. +# attribute-naming-style. If left empty, class attribute names will be checked +# with the set naming style. #class-attribute-rgx= +# Naming style matching correct class constant names. +class-const-naming-style=UPPER_CASE + +# Regular expression matching correct class constant names. Overrides class- +# const-naming-style. If left empty, class constant names will be checked with +# the set naming style. +#class-const-rgx= + # Naming style matching correct class names. class-naming-style=PascalCase # Regular expression matching correct class names. Overrides class-naming- -# style. +# style. If left empty, class names will be checked with the set naming style. #class-rgx= # Naming style matching correct constant names. const-naming-style=UPPER_CASE # Regular expression matching correct constant names. Overrides const-naming- +# style. If left empty, constant names will be checked with the set naming # style. #const-rgx= @@ -418,7 +180,8 @@ docstring-min-length=-1 function-naming-style=snake_case # Regular expression matching correct function names. Overrides function- -# naming-style. +# naming-style. If left empty, function names will be checked with the set +# naming style. #function-rgx= # Good variable names which should always be accepted, separated by a comma. @@ -440,21 +203,22 @@ include-naming-hint=no inlinevar-naming-style=any # Regular expression matching correct inline iteration names. Overrides -# inlinevar-naming-style. +# inlinevar-naming-style. If left empty, inline iteration names will be checked +# with the set naming style. #inlinevar-rgx= # Naming style matching correct method names. method-naming-style=snake_case # Regular expression matching correct method names. Overrides method-naming- -# style. +# style. If left empty, method names will be checked with the set naming style. #method-rgx= # Naming style matching correct module names. module-naming-style=snake_case # Regular expression matching correct module names. Overrides module-naming- -# style. +# style. If left empty, module names will be checked with the set naming style. #module-rgx= # Colon-delimited sets of names that determine each other's naming style when @@ -470,90 +234,56 @@ no-docstring-rgx=^_ # These decorators are taken in consideration only for invalid-name. property-classes=abc.abstractproperty +# Regular expression matching correct type alias names. If left empty, type +# alias names will be checked with the set naming style. +#typealias-rgx= + +# Regular expression matching correct type variable names. If left empty, type +# variable names will be checked with the set naming style. +#typevar-rgx= + # Naming style matching correct variable names. variable-naming-style=snake_case # Regular expression matching correct variable names. Overrides variable- -# naming-style. +# naming-style. If left empty, variable names will be checked with the set +# naming style. #variable-rgx= -[STRING] - -# This flag controls whether inconsistent-quotes generates a warning when the -# character used as a quote delimiter is used inconsistently within a module. -check-quote-consistency=no +[CLASSES] -# This flag controls whether the implicit-str-concat should generate a warning -# on implicit string concatenation in sequences defined over several lines. -check-str-concat-over-line-jumps=no - - -[IMPORTS] - -# List of modules that can be imported at any level, not just the top level -# one. -allow-any-import-level= - -# Allow wildcard imports from modules that define __all__. -allow-wildcard-with-all=no - -# Analyse import fallback blocks. This can be used to support both Python 2 and -# 3 compatible code, which means that the block might have code that exists -# only in one or another interpreter, leading to false positives when analysed. -analyse-fallback-blocks=no - -# Deprecated modules which should not be used, separated by a comma. -deprecated-modules=optparse,tkinter.tix - -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled). -ext-import-graph= - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled). -import-graph= - -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled). -int-import-graph= - -# Force import order to recognize a module as part of the standard -# compatibility libraries. -known-standard-library= - -# Force import order to recognize a module as part of a third party library. -known-third-party=enchant - -# Couples of modules and preferred modules, separated by a comma. -preferred-modules= - - -[CLASSES] +# Warn about protected attribute access inside special methods +check-protected-access-in-special-methods=no # List of method names used to declare (i.e. assign) instance attributes. defining-attr-methods=__init__, __new__, setUp, + asyncSetUp, __post_init__ # List of member names, which should be excluded from the protected access # warning. -exclude-protected=_asdict, - _fields, - _replace, - _source, - _make +exclude-protected=_asdict,_fields,_replace,_source,_make,os._exit # List of valid names for the first argument in a class method. valid-classmethod-first-arg=cls # List of valid names for the first argument in a metaclass class method. -valid-metaclass-classmethod-first-arg=cls +valid-metaclass-classmethod-first-arg=mcs [DESIGN] +# List of regular expressions of class ancestor names to ignore when counting +# public methods (see R0903) +exclude-too-few-public-methods= + +# List of qualified class names to ignore when counting class parents (see +# R0901) +ignored-parents= + # Maximum number of arguments for function / method. max-args=8 @@ -587,7 +317,320 @@ min-public-methods=2 [EXCEPTIONS] -# Exceptions that will emit a warning when being caught. Defaults to -# "BaseException, Exception". -overgeneral-exceptions=BaseException, - Exception +# Exceptions that will emit a warning when caught. +overgeneral-exceptions=builtins.BaseException,builtins.Exception + + +[FORMAT] + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=100 + +# Maximum number of lines in a module. +max-module-lines=2500 + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[IMPORTS] + +# List of modules that can be imported at any level, not just the top level +# one. +allow-any-import-level= + +# Allow explicit reexports by alias from a package __init__. +allow-reexport-from-package=no + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Deprecated modules which should not be used, separated by a comma. +deprecated-modules= + +# Output a graph (.gv or any supported image format) of external dependencies +# to the given file (report RP0402 must not be disabled). +ext-import-graph= + +# Output a graph (.gv or any supported image format) of all (i.e. internal and +# external) dependencies to the given file (report RP0402 must not be +# disabled). +import-graph= + +# Output a graph (.gv or any supported image format) of internal dependencies +# to the given file (report RP0402 must not be disabled). +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Couples of modules and preferred modules, separated by a comma. +preferred-modules= + + +[LOGGING] + +# The type of string formatting that logging methods do. `old` means using % +# formatting, `new` is for `{}` formatting. +logging-format-style=old + +# Logging modules to check that the string format arguments are in logging +# function parameter format. +logging-modules=logging + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, +# UNDEFINED. +confidence=HIGH, + CONTROL_FLOW, + INFERENCE, + INFERENCE_FAILURE, + UNDEFINED + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then re-enable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable=raw-checker-failed, + bad-inline-option, + locally-disabled, + file-ignored, + suppressed-message, + useless-suppression, + deprecated-pragma, + use-symbolic-message-instead, + consider-using-f-string, + no-member, + no-else-return, + no-else-raise, + inconsistent-return-statements + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable=c-extension-no-member + + +[METHOD_ARGS] + +# List of qualified names (i.e., library.method) which require a timeout +# parameter e.g. 'requests.api.get,requests.api.post' +timeout-methods=requests.api.delete,requests.api.get,requests.api.head,requests.api.options,requests.api.patch,requests.api.post,requests.api.put,requests.api.request + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME, + XXX, + TODO + +# Regular expression of note tags to take in consideration. +notes-rgx= + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=sys.exit,argparse.parse_error + + +[REPORTS] + +# Python expression which should return a score less than or equal to 10. You +# have access to the variables 'fatal', 'error', 'warning', 'refactor', +# 'convention', and 'info' which contain the number of messages in each +# category, as well as 'statement' which is the total number of statements +# analyzed. This score is used by the global evaluation report (RP0004). +evaluation=max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +msg-template= + +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +#output-format= + +# Tells whether to display a full report or only the messages. +reports=no + +# Activate the evaluation score. +score=yes + + +[SIMILARITIES] + +# Comments are removed from the similarity computation +ignore-comments=yes + +# Docstrings are removed from the similarity computation +ignore-docstrings=yes + +# Imports are removed from the similarity computation +ignore-imports=yes + +# Signatures are removed from the similarity computation +ignore-signatures=yes + +# Minimum lines number of a similarity. +min-similarity-lines=20 + + +[SPELLING] + +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions=4 + +# Spelling dictionary name. No available dictionaries : You need to install +# both the python package and the system dependency for enchant to work.. +spelling-dict= + +# List of comma separated words that should be considered directives if they +# appear at the beginning of a comment and should not be checked. +spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy: + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains the private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to the private dictionary (see the +# --spelling-private-dict-file option) instead of raising a message. +spelling-store-unknown-words=no + + +[STRING] + +# This flag controls whether inconsistent-quotes generates a warning when the +# character used as a quote delimiter is used inconsistently within a module. +check-quote-consistency=no + +# This flag controls whether the implicit-str-concat should generate a warning +# on implicit string concatenation in sequences defined over several lines. +check-str-concat-over-line-jumps=no + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# Tells whether to warn about missing members when the owner of the attribute +# is inferred to be None. +ignore-none=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes + +# List of symbolic message names to ignore for Mixin members. +ignored-checks-for-mixins=no-member, + not-async-context-manager, + not-context-manager, + attribute-defined-outside-init + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local,argparse.Namespace + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + +# Regex pattern to define which classes are considered mixins. +mixin-class-rgx=.*[Mm]ixin + +# List of decorators that change the signature of a decorated function. +signature-mutators= + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid defining new builtins when possible. +additional-builtins= + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes + +# List of names allowed to shadow builtins +allowed-redefined-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_, + _cb + +# A regular expression matching the name of dummy variables (i.e. expected to +# not be used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io diff --git a/diskcache/core.py b/diskcache/core.py index 05a0854..9f3a597 100644 --- a/diskcache/core.py +++ b/diskcache/core.py @@ -170,7 +170,7 @@ def get(self, key, raw): :return: corresponding Python key """ - # pylint: disable=no-self-use,unidiomatic-typecheck + # pylint: disable=unidiomatic-typecheck if raw: return bytes(key) if type(key) is sqlite3.Binary else key else: @@ -228,7 +228,6 @@ def store(self, value, read, key=UNKNOWN): return len(result), MODE_PICKLE, filename, None def _write(self, full_path, iterator, mode, encoding=None): - # pylint: disable=no-self-use full_dir, _ = op.split(full_path) for count in range(1, 11): @@ -264,7 +263,7 @@ def fetch(self, mode, filename, value, read): :raises: IOError if the value cannot be read """ - # pylint: disable=no-self-use,unidiomatic-typecheck,consider-using-with + # pylint: disable=unidiomatic-typecheck,consider-using-with if mode == MODE_RAW: return bytes(value) if type(value) is sqlite3.Binary else value elif mode == MODE_BINARY: @@ -1378,6 +1377,7 @@ def delete(self, key, retry=False): :raises Timeout: if database timeout occurs """ + # pylint: disable=unnecessary-dunder-call try: return self.__delitem__(key, retry=retry) except KeyError: From 471aa5e551aed186dd34d5bc7d345be835efd6f3 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 10 Apr 2023 23:26:38 -0700 Subject: [PATCH 105/126] Drop Python 3.7 from testing --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 3735ebc..e7217a7 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist=bluecheck,doc8,docs,isortcheck,flake8,mypy,pylint,rstcheck,py37,py38,py39,py310,py311 +envlist=bluecheck,doc8,docs,isortcheck,flake8,mypy,pylint,rstcheck,py38,py39,py310,py311 skip_missing_interpreters=True [testenv] From c14345f105f14eba45986b09ec96d87b8997c9cc Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 10 Apr 2023 23:42:38 -0700 Subject: [PATCH 106/126] Update tests for Django 4.2 --- tests/test_djangocache.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_djangocache.py b/tests/test_djangocache.py index b5cc2a8..734ba1b 100644 --- a/tests/test_djangocache.py +++ b/tests/test_djangocache.py @@ -870,7 +870,6 @@ def test_custom_key_func(self): def test_cache_write_unpicklable_object(self): fetch_middleware = FetchFromCacheMiddleware(empty_response) - fetch_middleware.cache = cache request = self.factory.get('/cache/test') request._cache_update_cache = True @@ -887,7 +886,6 @@ def get_response(req): return response update_middleware = UpdateCacheMiddleware(get_response) - update_middleware.cache = cache response = update_middleware(request) get_cache_data = fetch_middleware.process_request(request) From 0294d58cd0cb72dd6affbbc46b3ff05d96d015cc Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 10 Apr 2023 23:43:15 -0700 Subject: [PATCH 107/126] Bump version to v5.5.0 --- diskcache/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/diskcache/__init__.py b/diskcache/__init__.py index f7aa771..8d7e28c 100644 --- a/diskcache/__init__.py +++ b/diskcache/__init__.py @@ -61,8 +61,8 @@ pass __title__ = 'diskcache' -__version__ = '5.4.0' -__build__ = 0x050400 +__version__ = '5.5.0' +__build__ = 0x050500 __author__ = 'Grant Jenks' __license__ = 'Apache 2.0' __copyright__ = 'Copyright 2016-2023 Grant Jenks' From 6cd6888a16be1d531a19ca91d5e0daa6edac9718 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 10 Apr 2023 23:45:14 -0700 Subject: [PATCH 108/126] Drop 3.7 from CI --- .github/workflows/integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 2d83ad3..07aceec 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -32,7 +32,7 @@ jobs: max-parallel: 8 matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: [3.7, 3.8, 3.9, '3.10', 3.11] + python-version: [3.8, 3.9, '3.10', 3.11] steps: - name: Set up Python ${{ matrix.python-version }} x64 From bbac13b54f9ac0a5c45b7cfbd242869b41042cac Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 10 Apr 2023 23:52:57 -0700 Subject: [PATCH 109/126] Install dev requirements for wheel package --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 676593c..21b6e5b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -26,7 +26,7 @@ jobs: - name: Install dependencies run: | pip install --upgrade pip - pip install -r requirements.txt + pip install -r requirements-dev.txt - name: Create source dist run: python setup.py sdist From 0f5d8ed63406e26de96701f98aa585b4fb26f6dd Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 10 Apr 2023 23:53:20 -0700 Subject: [PATCH 110/126] Bump version to 5.5.1 --- diskcache/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/diskcache/__init__.py b/diskcache/__init__.py index 8d7e28c..95dafb3 100644 --- a/diskcache/__init__.py +++ b/diskcache/__init__.py @@ -61,8 +61,8 @@ pass __title__ = 'diskcache' -__version__ = '5.5.0' -__build__ = 0x050500 +__version__ = '5.5.1' +__build__ = 0x050501 __author__ = 'Grant Jenks' __license__ = 'Apache 2.0' __copyright__ = 'Copyright 2016-2023 Grant Jenks' From fe5ee43ac5df1847556f280c696ab921f0910be2 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Sun, 16 Apr 2023 21:42:13 -0700 Subject: [PATCH 111/126] Close the cache explicitly before deleting the reference --- diskcache/persistent.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/diskcache/persistent.py b/diskcache/persistent.py index c3d570b..cce3736 100644 --- a/diskcache/persistent.py +++ b/diskcache/persistent.py @@ -533,12 +533,13 @@ def reverse(self): # GrantJ 2019-03-22 Consider using an algorithm that swaps the values # at two keys. Like self._cache.swap(key1, key2, retry=True) The swap # method would exchange the values at two given keys. Then, using a - # forward iterator and a reverse iterator, the reversis method could + # forward iterator and a reverse iterator, the reverse method could # avoid making copies of the values. temp = Deque(iterable=reversed(self)) self.clear() self.extend(temp) directory = temp.directory + temp.close() del temp rmtree(directory) From 9380c784d9e2954611b2ea309f1c657602085f25 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Sun, 16 Apr 2023 23:02:24 -0700 Subject: [PATCH 112/126] Oops, close the cache, not the deque --- diskcache/persistent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/diskcache/persistent.py b/diskcache/persistent.py index cce3736..01cf4e3 100644 --- a/diskcache/persistent.py +++ b/diskcache/persistent.py @@ -539,7 +539,7 @@ def reverse(self): self.clear() self.extend(temp) directory = temp.directory - temp.close() + temp._cache.close() del temp rmtree(directory) From f5a17ff0959a4cc7147d45a4cc8d8eef8d7416b0 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 17 Apr 2023 19:00:34 -0700 Subject: [PATCH 113/126] Shutup pylint --- diskcache/persistent.py | 1 + 1 file changed, 1 insertion(+) diff --git a/diskcache/persistent.py b/diskcache/persistent.py index 01cf4e3..c3f22b5 100644 --- a/diskcache/persistent.py +++ b/diskcache/persistent.py @@ -530,6 +530,7 @@ def reverse(self): ['c', 'b', 'a'] """ + # pylint: disable=protected-access # GrantJ 2019-03-22 Consider using an algorithm that swaps the values # at two keys. Like self._cache.swap(key1, key2, retry=True) The swap # method would exchange the values at two given keys. Then, using a From ef94856d2447fa9662bc62557f9b96ba6f131e15 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 17 Apr 2023 19:01:25 -0700 Subject: [PATCH 114/126] Bump version to 5.5.2 --- diskcache/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/diskcache/__init__.py b/diskcache/__init__.py index 95dafb3..134c88a 100644 --- a/diskcache/__init__.py +++ b/diskcache/__init__.py @@ -61,8 +61,8 @@ pass __title__ = 'diskcache' -__version__ = '5.5.1' -__build__ = 0x050501 +__version__ = '5.5.2' +__build__ = 0x050502 __author__ = 'Grant Jenks' __license__ = 'Apache 2.0' __copyright__ = 'Copyright 2016-2023 Grant Jenks' From 74e554c5d9340765f6fd6f7891da49a420e92b70 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 17 Apr 2023 19:05:01 -0700 Subject: [PATCH 115/126] Bump versions of checkout and setup-python --- .github/workflows/integration.yml | 8 ++++---- .github/workflows/release.yml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 07aceec..b596fc6 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -12,9 +12,9 @@ jobs: check: [bluecheck, doc8, docs, flake8, isortcheck, mypy, pylint, rstcheck] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: '3.11' - name: Install dependencies @@ -36,12 +36,12 @@ jobs: steps: - name: Set up Python ${{ matrix.python-version }} x64 - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} architecture: x64 - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Install tox run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 21b6e5b..efe73c6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Install libmemcached-dev run: | @@ -19,7 +19,7 @@ jobs: sudo apt-get install libmemcached-dev - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: '3.11' From 35dbeabd283b242e9afd33713a5cea5cd260f51d Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 17 Apr 2023 20:38:04 -0700 Subject: [PATCH 116/126] Add maxlen parameter to diskcache.Deque (#191) * Add maxlen parameter to diskcache.Deque --- diskcache/djangocache.py | 5 ++- diskcache/fanout.py | 5 ++- diskcache/persistent.py | 91 +++++++++++++++++++++++++++++++--------- docs/tutorial.rst | 3 ++ tests/test_deque.py | 27 +++++++++++- 5 files changed, 107 insertions(+), 24 deletions(-) diff --git a/diskcache/djangocache.py b/diskcache/djangocache.py index 8bf85ce..5dc8ce2 100644 --- a/diskcache/djangocache.py +++ b/diskcache/djangocache.py @@ -44,14 +44,15 @@ def cache(self, name): """ return self._cache.cache(name) - def deque(self, name): + def deque(self, name, maxlen=None): """Return Deque with given `name` in subdirectory. :param str name: subdirectory name for Deque + :param maxlen: max length (default None, no max) :return: Deque with given name """ - return self._cache.deque(name) + return self._cache.deque(name, maxlen=maxlen) def index(self, name): """Return Index with given `name` in subdirectory. diff --git a/diskcache/fanout.py b/diskcache/fanout.py index 5283490..50005fc 100644 --- a/diskcache/fanout.py +++ b/diskcache/fanout.py @@ -612,7 +612,7 @@ def cache(self, name, timeout=60, disk=None, **settings): _caches[name] = temp return temp - def deque(self, name): + def deque(self, name, maxlen=None): """Return Deque with given `name` in subdirectory. >>> cache = FanoutCache() @@ -626,6 +626,7 @@ def deque(self, name): 1 :param str name: subdirectory name for Deque + :param maxlen: max length (default None, no max) :return: Deque with given name """ @@ -641,7 +642,7 @@ def deque(self, name): disk=self._disk, eviction_policy='none', ) - deque = Deque.fromcache(cache) + deque = Deque.fromcache(cache, maxlen=maxlen) _deques[name] = deque return deque diff --git a/diskcache/persistent.py b/diskcache/persistent.py index c3f22b5..522bb74 100644 --- a/diskcache/persistent.py +++ b/diskcache/persistent.py @@ -75,7 +75,7 @@ class Deque(Sequence): """ - def __init__(self, iterable=(), directory=None): + def __init__(self, iterable=(), directory=None, maxlen=None): """Initialize deque instance. If directory is None then temporary directory created. The directory @@ -86,10 +86,11 @@ def __init__(self, iterable=(), directory=None): """ self._cache = Cache(directory, eviction_policy='none') - self.extend(iterable) + self._maxlen = float('inf') if maxlen is None else maxlen + self._extend(iterable) @classmethod - def fromcache(cls, cache, iterable=()): + def fromcache(cls, cache, iterable=(), maxlen=None): """Initialize deque using `cache`. >>> cache = Cache() @@ -111,7 +112,8 @@ def fromcache(cls, cache, iterable=()): # pylint: disable=no-member,protected-access self = cls.__new__(cls) self._cache = cache - self.extend(iterable) + self._maxlen = float('inf') if maxlen is None else maxlen + self._extend(iterable) return self @property @@ -124,6 +126,31 @@ def directory(self): """Directory path where deque is stored.""" return self._cache.directory + @property + def maxlen(self): + """Max length of the deque.""" + return self._maxlen + + @maxlen.setter + def maxlen(self, value): + """Set max length of the deque. + + Pops items from left while length greater than max. + + >>> deque = Deque() + >>> deque.extendleft('abcde') + >>> deque.maxlen = 3 + >>> list(deque) + ['c', 'd', 'e'] + + :param value: max length + + """ + self._maxlen = value + with self._cache.transact(retry=True): + while len(self._cache) > self._maxlen: + self._popleft() + def _index(self, index, func): len_self = len(self) @@ -244,7 +271,7 @@ def __iadd__(self, iterable): :return: deque with added items """ - self.extend(iterable) + self._extend(iterable) return self def __iter__(self): @@ -292,10 +319,11 @@ def __reversed__(self): pass def __getstate__(self): - return self.directory + return self.directory, self.maxlen def __setstate__(self, state): - self.__init__(directory=state) + directory, maxlen = state + self.__init__(directory=directory, maxlen=maxlen) def append(self, value): """Add `value` to back of deque. @@ -310,7 +338,12 @@ def append(self, value): :param value: value to add to back of deque """ - self._cache.push(value, retry=True) + with self._cache.transact(retry=True): + self._cache.push(value, retry=True) + if len(self._cache) > self._maxlen: + self._popleft() + + _append = append def appendleft(self, value): """Add `value` to front of deque. @@ -325,7 +358,12 @@ def appendleft(self, value): :param value: value to add to front of deque """ - self._cache.push(value, side='front', retry=True) + with self._cache.transact(retry=True): + self._cache.push(value, side='front', retry=True) + if len(self._cache) > self._maxlen: + self._pop() + + _appendleft = appendleft def clear(self): """Remove all elements from deque. @@ -340,6 +378,13 @@ def clear(self): """ self._cache.clear(retry=True) + _clear = clear + + def copy(self): + """Copy deque with same directory and max length.""" + TypeSelf = type(self) + return TypeSelf(directory=self.directory, maxlen=self.maxlen) + def count(self, value): """Return number of occurrences of `value` in deque. @@ -365,7 +410,9 @@ def extend(self, iterable): """ for value in iterable: - self.append(value) + self._append(value) + + _extend = extend def extendleft(self, iterable): """Extend front side of deque with value from `iterable`. @@ -379,7 +426,7 @@ def extendleft(self, iterable): """ for value in iterable: - self.appendleft(value) + self._appendleft(value) def peek(self): """Peek at value at back of deque. @@ -459,6 +506,8 @@ def pop(self): raise IndexError('pop from an empty deque') return value + _pop = pop + def popleft(self): """Remove and return value at front of deque. @@ -483,6 +532,8 @@ def popleft(self): raise IndexError('pop from an empty deque') return value + _popleft = popleft + def remove(self, value): """Remove first occurrence of `value` in deque. @@ -537,8 +588,8 @@ def reverse(self): # forward iterator and a reverse iterator, the reverse method could # avoid making copies of the values. temp = Deque(iterable=reversed(self)) - self.clear() - self.extend(temp) + self._clear() + self._extend(temp) directory = temp.directory temp._cache.close() del temp @@ -575,22 +626,22 @@ def rotate(self, steps=1): for _ in range(steps): try: - value = self.pop() + value = self._pop() except IndexError: return else: - self.appendleft(value) + self._appendleft(value) else: steps *= -1 steps %= len_self for _ in range(steps): try: - value = self.popleft() + value = self._popleft() except IndexError: return else: - self.append(value) + self._append(value) __hash__ = None # type: ignore @@ -669,7 +720,9 @@ def __init__(self, *args, **kwargs): args = args[1:] directory = None self._cache = Cache(directory, eviction_policy='none') - self.update(*args, **kwargs) + self._update(*args, **kwargs) + + _update = MutableMapping.update @classmethod def fromcache(cls, cache, *args, **kwargs): @@ -695,7 +748,7 @@ def fromcache(cls, cache, *args, **kwargs): # pylint: disable=no-member,protected-access self = cls.__new__(cls) self._cache = cache - self.update(*args, **kwargs) + self._update(*args, **kwargs) return self @property diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 1963635..69277d3 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -565,6 +565,9 @@ access and editing at both front and back sides. :class:`Deque 4 >>> other.popleft() 'foo' + >>> thing = Deque('abcde', maxlen=3) + >>> list(thing) + ['c', 'd', 'e'] :class:`Deque ` objects provide an efficient and safe means of cross-thread and cross-process communication. :class:`Deque ` diff --git a/tests/test_deque.py b/tests/test_deque.py index add7714..71c69e2 100644 --- a/tests/test_deque.py +++ b/tests/test_deque.py @@ -77,6 +77,20 @@ def test_getsetdel(deque): assert len(deque) == 0 +def test_append(deque): + deque.maxlen = 3 + for item in 'abcde': + deque.append(item) + assert deque == 'cde' + + +def test_appendleft(deque): + deque.maxlen = 3 + for item in 'abcde': + deque.appendleft(item) + assert deque == 'edc' + + def test_index_positive(deque): cache = mock.MagicMock() cache.__len__.return_value = 3 @@ -131,9 +145,12 @@ def test_state(deque): sequence = list('abcde') deque.extend(sequence) assert deque == sequence + deque.maxlen = 3 + assert list(deque) == sequence[-3:] state = pickle.dumps(deque) values = pickle.loads(state) - assert values == sequence + assert values == sequence[-3:] + assert values.maxlen == 3 def test_compare(deque): @@ -161,6 +178,14 @@ def test_repr(): assert repr(deque) == 'Deque(directory=%r)' % directory +def test_copy(deque): + sequence = list('abcde') + deque.extend(sequence) + temp = deque.copy() + assert deque == sequence + assert temp == sequence + + def test_count(deque): deque += 'abbcccddddeeeee' From 4beffe892a6c4352098a79614de40649d7e9f88e Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 17 Apr 2023 20:40:55 -0700 Subject: [PATCH 117/126] Bump version to 5.6.0 --- diskcache/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/diskcache/__init__.py b/diskcache/__init__.py index 134c88a..8647b9a 100644 --- a/diskcache/__init__.py +++ b/diskcache/__init__.py @@ -61,8 +61,8 @@ pass __title__ = 'diskcache' -__version__ = '5.5.2' -__build__ = 0x050502 +__version__ = '5.6.0' +__build__ = 0x050600 __author__ = 'Grant Jenks' __license__ = 'Apache 2.0' __copyright__ = 'Copyright 2016-2023 Grant Jenks' From cffbcec2b198e3a296ec294bd43da37fc559645b Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 17 Apr 2023 22:30:26 -0700 Subject: [PATCH 118/126] Fix docs re: JSONDisk --- docs/tutorial.rst | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 69277d3..2eb454d 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -821,30 +821,28 @@ example below uses compressed JSON, available for convenience as .. code-block:: python - import json, zlib - class JSONDisk(diskcache.Disk): def __init__(self, directory, compress_level=1, **kwargs): self.compress_level = compress_level - super(JSONDisk, self).__init__(directory, **kwargs) + super().__init__(directory, **kwargs) def put(self, key): json_bytes = json.dumps(key).encode('utf-8') data = zlib.compress(json_bytes, self.compress_level) - return super(JSONDisk, self).put(data) + return super().put(data) def get(self, key, raw): - data = super(JSONDisk, self).get(key, raw) + data = super().get(key, raw) return json.loads(zlib.decompress(data).decode('utf-8')) - def store(self, value, read): + def store(self, value, read, key=UNKNOWN): if not read: json_bytes = json.dumps(value).encode('utf-8') value = zlib.compress(json_bytes, self.compress_level) - return super(JSONDisk, self).store(value, read) + return super().store(value, read, key=key) def fetch(self, mode, filename, value, read): - data = super(JSONDisk, self).fetch(mode, filename, value, read) + data = super().fetch(mode, filename, value, read) if not read: data = json.loads(zlib.decompress(data).decode('utf-8')) return data From f81160f22af9e8af0e07e179808280188146a020 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 17 Apr 2023 22:46:44 -0700 Subject: [PATCH 119/126] Support pathlib.Path as directory argument --- diskcache/core.py | 1 + diskcache/fanout.py | 1 + tests/test_core.py | 8 ++++++++ tests/test_fanout.py | 8 ++++++++ 4 files changed, 18 insertions(+) diff --git a/diskcache/core.py b/diskcache/core.py index 9f3a597..af65454 100644 --- a/diskcache/core.py +++ b/diskcache/core.py @@ -433,6 +433,7 @@ def __init__(self, directory=None, timeout=60, disk=Disk, **settings): if directory is None: directory = tempfile.mkdtemp(prefix='diskcache-') + directory = str(directory) directory = op.expanduser(directory) directory = op.expandvars(directory) diff --git a/diskcache/fanout.py b/diskcache/fanout.py index 50005fc..9822ee4 100644 --- a/diskcache/fanout.py +++ b/diskcache/fanout.py @@ -30,6 +30,7 @@ def __init__( """ if directory is None: directory = tempfile.mkdtemp(prefix='diskcache-') + directory = str(directory) directory = op.expanduser(directory) directory = op.expandvars(directory) diff --git a/tests/test_core.py b/tests/test_core.py index 356d104..788afef 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -5,6 +5,7 @@ import io import os import os.path as op +import pathlib import pickle import shutil import sqlite3 @@ -37,6 +38,13 @@ def test_init(cache): cache.close() +def test_init_path(cache): + path = pathlib.Path(cache.directory) + other = dc.Cache(path) + other.close() + assert cache.directory == other.directory + + def test_init_disk(): with dc.Cache(disk_pickle_protocol=1, disk_min_file_size=2**20) as cache: key = (None, 0, 'abc') diff --git a/tests/test_fanout.py b/tests/test_fanout.py index deea03f..af221b6 100644 --- a/tests/test_fanout.py +++ b/tests/test_fanout.py @@ -5,6 +5,7 @@ import io import os import os.path as op +import pathlib import pickle import shutil import subprocess as sp @@ -44,6 +45,13 @@ def test_init(cache): cache.check() +def test_init_path(cache): + path = pathlib.Path(cache.directory) + other = dc.FanoutCache(path) + other.close() + assert cache.directory == other.directory + + def test_set_get_delete(cache): for value in range(100): cache.set(value, value) From 4d3068625a3edcd2f5a1f6f104ef621f1f7ea395 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 17 Apr 2023 22:47:22 -0700 Subject: [PATCH 120/126] Bump version to 5.6.1 --- diskcache/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/diskcache/__init__.py b/diskcache/__init__.py index 8647b9a..1931a0d 100644 --- a/diskcache/__init__.py +++ b/diskcache/__init__.py @@ -61,8 +61,8 @@ pass __title__ = 'diskcache' -__version__ = '5.6.0' -__build__ = 0x050600 +__version__ = '5.6.1' +__build__ = 0x050601 __author__ = 'Grant Jenks' __license__ = 'Apache 2.0' __copyright__ = 'Copyright 2016-2023 Grant Jenks' From 17a5f42facc312dae6e98b7b53345e2ed02be21d Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Wed, 30 Aug 2023 22:56:38 -0700 Subject: [PATCH 121/126] Bug fix: Fix peek when value is so large that a file is used (#288) Error caused by copy/paste from pull(). --- diskcache/core.py | 3 --- tests/test_deque.py | 10 ++++++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/diskcache/core.py b/diskcache/core.py index af65454..c7c8486 100644 --- a/diskcache/core.py +++ b/diskcache/core.py @@ -1703,9 +1703,6 @@ def peek( except IOError: # Key was deleted before we could retrieve result. continue - finally: - if name is not None: - self._disk.remove(name) break if expire_time and tag: diff --git a/tests/test_deque.py b/tests/test_deque.py index 71c69e2..f997a86 100644 --- a/tests/test_deque.py +++ b/tests/test_deque.py @@ -302,3 +302,13 @@ def test_rotate_indexerror_negative(deque): with mock.patch.object(deque, '_cache', cache): deque.rotate(-1) + + +def test_peek(deque): + value = b'x' * 100_000 + deque.append(value) + assert len(deque) == 1 + assert deque.peek() == value + assert len(deque) == 1 + assert deque.peek() == value + assert len(deque) == 1 From 63a5f6068b77fe9c02c8f310758fa1f05ae1ae04 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Wed, 30 Aug 2023 22:58:11 -0700 Subject: [PATCH 122/126] Bump version to 5.6.2 --- diskcache/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/diskcache/__init__.py b/diskcache/__init__.py index 1931a0d..719640f 100644 --- a/diskcache/__init__.py +++ b/diskcache/__init__.py @@ -61,8 +61,8 @@ pass __title__ = 'diskcache' -__version__ = '5.6.1' -__build__ = 0x050601 +__version__ = '5.6.2' +__build__ = 0x050602 __author__ = 'Grant Jenks' __license__ = 'Apache 2.0' __copyright__ = 'Copyright 2016-2023 Grant Jenks' From 23d10dce8f4be9c00df4786d508964b3b7d72b27 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Wed, 30 Aug 2023 23:09:54 -0700 Subject: [PATCH 123/126] Update release.yml to use pypa/gh-action-pypi-publish --- .github/workflows/release.yml | 29 ++++++++--------------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index efe73c6..33b3a8f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,35 +9,22 @@ jobs: upload: runs-on: ubuntu-latest + permissions: + id-token: write steps: - uses: actions/checkout@v3 - - name: Install libmemcached-dev - run: | - sudo apt-get update - sudo apt-get install libmemcached-dev - - name: Set up Python uses: actions/setup-python@v4 with: python-version: '3.11' - - name: Install dependencies - run: | - pip install --upgrade pip - pip install -r requirements-dev.txt - - - name: Create source dist - run: python setup.py sdist + - name: Install build + run: pip install build - - name: Create wheel dist - run: python setup.py bdist_wheel + - name: Create build + run: python -m build - - name: Upload with twine - env: - TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }} - TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} - run: | - ls -l dist/* - twine upload dist/* + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 From 323787f507a6456c56cce213156a78b17073fe00 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Wed, 30 Aug 2023 23:10:27 -0700 Subject: [PATCH 124/126] Bump version to 5.6.3 --- diskcache/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/diskcache/__init__.py b/diskcache/__init__.py index 719640f..7757d66 100644 --- a/diskcache/__init__.py +++ b/diskcache/__init__.py @@ -61,8 +61,8 @@ pass __title__ = 'diskcache' -__version__ = '5.6.2' -__build__ = 0x050602 +__version__ = '5.6.3' +__build__ = 0x050603 __author__ = 'Grant Jenks' __license__ = 'Apache 2.0' __copyright__ = 'Copyright 2016-2023 Grant Jenks' From 9cd3816333fa34cb30d6cc2a7f227b6b1cdb793c Mon Sep 17 00:00:00 2001 From: ddorian Date: Tue, 27 Feb 2024 00:30:46 +0100 Subject: [PATCH 125/126] Change `Cache_expire_time` to a partial index because we don't need to query rows efficiently `where expire_time IS NULL` (#305) --- diskcache/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/diskcache/core.py b/diskcache/core.py index c7c8486..ad9ad4c 100644 --- a/diskcache/core.py +++ b/diskcache/core.py @@ -531,7 +531,7 @@ def __init__(self, directory=None, timeout=60, disk=Disk, **settings): sql( 'CREATE INDEX IF NOT EXISTS Cache_expire_time ON' - ' Cache (expire_time)' + ' Cache (expire_time) WHERE expire_time IS NOT NULL' ) query = EVICTION_POLICY[self.eviction_policy]['init'] From ebfa37cd99d7ef716ec452ad8af4b4276a8e2233 Mon Sep 17 00:00:00 2001 From: ddorian Date: Sun, 3 Mar 2024 02:19:29 +0100 Subject: [PATCH 126/126] Change `Cache_tag_rowid` to a partial index because we don't need to query rows efficiently `where tag IS NULL` (#307) * Change `Cache_tag_rowid` to a partial index because we don't need to query rows efficiently `where tag IS NULL` * Fix formatting --- diskcache/core.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/diskcache/core.py b/diskcache/core.py index ad9ad4c..7a3d23b 100644 --- a/diskcache/core.py +++ b/diskcache/core.py @@ -2028,7 +2028,10 @@ def create_tag_index(self): """ sql = self._sql - sql('CREATE INDEX IF NOT EXISTS Cache_tag_rowid ON Cache(tag, rowid)') + sql( + 'CREATE INDEX IF NOT EXISTS Cache_tag_rowid ON Cache(tag, rowid) ' + 'WHERE tag IS NOT NULL' + ) self.reset('tag_index', 1) def drop_tag_index(self):