From c1774469b8d4c4906fe24f7b5afd637795af48d9 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 10 Apr 2023 22:12:28 -0700 Subject: [PATCH 01/31] Remove unused import --- setup.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/setup.py b/setup.py index b49d9c8..841dfb9 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,3 @@ -from io import open - from setuptools import setup from setuptools.command.test import test as TestCommand From f3fcdffee88af7923740b9864d533524a79d1222 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 10 Apr 2023 22:27:48 -0700 Subject: [PATCH 02/31] Update Cache(...) params when allocating --- diskcache/fanout.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/diskcache/fanout.py b/diskcache/fanout.py index dc5240c..8fe51d9 100644 --- a/diskcache/fanout.py +++ b/diskcache/fanout.py @@ -573,9 +573,11 @@ def reset(self, key, value=ENOVAL): break return result - def cache(self, name): + def cache(self, name, timeout=60, disk=None, **settings): """Return Cache with given `name` in subdirectory. + If disk is none (default), uses the fanout cache disk. + >>> fanout_cache = FanoutCache() >>> cache = fanout_cache.cache('test') >>> cache.set('abc', 123) @@ -588,6 +590,9 @@ def cache(self, name): True :param str name: subdirectory name for Cache + :param float timeout: SQLite connection timeout + :param disk: Disk type or subclass for serialization + :param settings: any of DEFAULT_SETTINGS :return: Cache with given name """ @@ -598,7 +603,12 @@ def cache(self, name): except KeyError: parts = name.split('/') directory = op.join(self._directory, 'cache', *parts) - temp = Cache(directory=directory, disk=self._disk) + temp = Cache( + directory=directory, + timeout=timeout, + disk=self._disk if disk is None else Disk, + **settings, + ) _caches[name] = temp return temp @@ -626,7 +636,11 @@ def deque(self, name): except KeyError: parts = name.split('/') directory = op.join(self._directory, 'deque', *parts) - cache = Cache(directory=directory, disk=self._disk) + cache = Cache( + directory=directory, + disk=self._disk, + eviction_policy='none', + ) deque = Deque.fromcache(cache) _deques[name] = deque return deque @@ -658,7 +672,11 @@ def index(self, name): except KeyError: parts = name.split('/') directory = op.join(self._directory, 'index', *parts) - cache = Cache(directory=directory, disk=self._disk) + cache = Cache( + directory=directory, + disk=self._disk, + eviction_policy='none', + ) index = Index.fromcache(cache) _indexes[name] = index return index From ee7a248e5c09e6fb9145b2e4a1777a345114b71d Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 10 Apr 2023 22:37:39 -0700 Subject: [PATCH 03/31] Add docs about the eviction policy to recipes --- diskcache/recipes.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/diskcache/recipes.py b/diskcache/recipes.py index b5af6dd..babb68f 100644 --- a/diskcache/recipes.py +++ b/diskcache/recipes.py @@ -17,6 +17,9 @@ class Averager: Sometimes known as "online statistics," the running average maintains the total and count. The average can then be calculated at any time. + Assumes the key will not be evicted. Set the eviction policy to 'none' on + the cache to guarantee the key is not evicted. + >>> import diskcache >>> cache = diskcache.FanoutCache() >>> ave = Averager(cache, 'latency') @@ -65,6 +68,9 @@ def pop(self): class Lock: """Recipe for cross-process and cross-thread lock. + Assumes the key will not be evicted. Set the eviction policy to 'none' on + the cache to guarantee the key is not evicted. + >>> import diskcache >>> cache = diskcache.Cache() >>> lock = Lock(cache, 'report-123') @@ -113,6 +119,9 @@ def __exit__(self, *exc_info): class RLock: """Recipe for cross-process and cross-thread re-entrant lock. + Assumes the key will not be evicted. Set the eviction policy to 'none' on + the cache to guarantee the key is not evicted. + >>> import diskcache >>> cache = diskcache.Cache() >>> rlock = RLock(cache, 'user-123') @@ -181,6 +190,9 @@ def __exit__(self, *exc_info): class BoundedSemaphore: """Recipe for cross-process and cross-thread bounded semaphore. + Assumes the key will not be evicted. Set the eviction policy to 'none' on + the cache to guarantee the key is not evicted. + >>> import diskcache >>> cache = diskcache.Cache() >>> semaphore = BoundedSemaphore(cache, 'max-cons', value=2) @@ -251,6 +263,9 @@ def throttle( ): """Decorator to throttle calls to function. + Assumes keys will not be evicted. Set the eviction policy to 'none' on the + cache to guarantee the keys are not evicted. + >>> import diskcache, time >>> cache = diskcache.Cache() >>> count = 0 @@ -305,6 +320,9 @@ def barrier(cache, lock_factory, name=None, expire=None, tag=None): Supports different kinds of locks: Lock, RLock, BoundedSemaphore. + Assumes keys will not be evicted. Set the eviction policy to 'none' on the + cache to guarantee the keys are not evicted. + >>> import diskcache, time >>> cache = diskcache.Cache() >>> @barrier(cache, Lock) From fb2fa2c401bb88ecbbc58009d02cbe65f2fc594a Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 10 Apr 2023 22:52:52 -0700 Subject: [PATCH 04/31] Test on Django 4.2 LTS --- tox.ini | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 36ed2b8..0ddcc8f 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,7 @@ skip_missing_interpreters=True [testenv] commands=pytest deps= - django==3.2.* + django==4.2.* pytest pytest-cov pytest-django @@ -31,7 +31,7 @@ allowlist_externals=make changedir=docs commands=make html deps= - django==3.2.* + django==4.2.* sphinx [testenv:flake8] @@ -53,7 +53,7 @@ deps=mypy [testenv:pylint] commands=pylint {toxinidir}/diskcache deps= - django==3.2.* + django==4.2.* pylint [testenv:rstcheck] From 0a9783353ff7dc9f874154ef98b23de27e4aba8d Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 10 Apr 2023 22:53:17 -0700 Subject: [PATCH 05/31] Update year to 2023 --- README.rst | 4 ++-- diskcache/__init__.py | 2 +- docs/conf.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index eb06a6a..04abdc0 100644 --- a/README.rst +++ b/README.rst @@ -4,7 +4,7 @@ DiskCache: Disk Backed Cache `DiskCache`_ is an Apache2 licensed disk and file backed cache library, written in pure-Python, and compatible with Django. -The cloud-based computing of 2021 puts a premium on memory. Gigabytes of empty +The cloud-based computing of 2023 puts a premium on memory. Gigabytes of empty space is left on disks as processes vie for memory. Among these processes is Memcached (and sometimes Redis) which is used as a cache. Wouldn't it be nice to leverage empty disk space for caching? @@ -387,7 +387,7 @@ Reference License ------- -Copyright 2016-2022 Grant Jenks +Copyright 2016-2023 Grant Jenks Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the diff --git a/diskcache/__init__.py b/diskcache/__init__.py index 2355128..f7aa771 100644 --- a/diskcache/__init__.py +++ b/diskcache/__init__.py @@ -65,4 +65,4 @@ __build__ = 0x050400 __author__ = 'Grant Jenks' __license__ = 'Apache 2.0' -__copyright__ = 'Copyright 2016-2022 Grant Jenks' +__copyright__ = 'Copyright 2016-2023 Grant Jenks' diff --git a/docs/conf.py b/docs/conf.py index 92ce1b9..92bf3ec 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -20,7 +20,7 @@ # -- Project information ----------------------------------------------------- project = 'DiskCache' -copyright = '2022, Grant Jenks' +copyright = '2023, Grant Jenks' author = 'Grant Jenks' # The full version, including alpha/beta/rc tags From 712cc1827b29fb8f6a76803d32f87a5fb57f3c6e Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 10 Apr 2023 22:56:16 -0700 Subject: [PATCH 06/31] Bump python testing to 3.11 --- .github/workflows/integration.yml | 4 ++-- .github/workflows/release.yml | 2 +- tox.ini | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 07a5650..2d83ad3 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -16,7 +16,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: '3.10' + python-version: '3.11' - name: Install dependencies run: | pip install --upgrade pip @@ -32,7 +32,7 @@ jobs: max-parallel: 8 matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: [3.6, 3.7, 3.8, 3.9, '3.10'] + python-version: [3.7, 3.8, 3.9, '3.10', 3.11] steps: - name: Set up Python ${{ matrix.python-version }} x64 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1f89c14..676593c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -21,7 +21,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: '3.10' + python-version: '3.11' - name: Install dependencies run: | diff --git a/tox.ini b/tox.ini index 0ddcc8f..3735ebc 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist=bluecheck,doc8,docs,isortcheck,flake8,mypy,pylint,rstcheck,py36,py37,py38,py39 +envlist=bluecheck,doc8,docs,isortcheck,flake8,mypy,pylint,rstcheck,py37,py38,py39,py310,py311 skip_missing_interpreters=True [testenv] From d7ae0990b240e8ea22c1a4060f58900edd339a18 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 10 Apr 2023 23:07:29 -0700 Subject: [PATCH 07/31] i blue it --- diskcache/core.py | 10 ++--- diskcache/fanout.py | 2 +- tests/benchmark_glob.py | 2 +- tests/settings.py | 2 +- tests/stress_test_core.py | 30 ++++++------- tests/stress_test_fanout.py | 30 ++++++------- tests/test_core.py | 84 ++++++++++++++++++------------------- tests/test_djangocache.py | 2 +- tests/test_fanout.py | 16 +++---- 9 files changed, 85 insertions(+), 93 deletions(-) diff --git a/diskcache/core.py b/diskcache/core.py index fb343be..05a0854 100644 --- a/diskcache/core.py +++ b/diskcache/core.py @@ -49,14 +49,14 @@ def __repr__(self): 'statistics': 0, # False 'tag_index': 0, # False 'eviction_policy': 'least-recently-stored', - 'size_limit': 2 ** 30, # 1gb + 'size_limit': 2**30, # 1gb 'cull_limit': 10, 'sqlite_auto_vacuum': 1, # FULL - 'sqlite_cache_size': 2 ** 13, # 8,192 pages + 'sqlite_cache_size': 2**13, # 8,192 pages 'sqlite_journal_mode': 'wal', - 'sqlite_mmap_size': 2 ** 26, # 64mb + 'sqlite_mmap_size': 2**26, # 64mb 'sqlite_synchronous': 1, # NORMAL - 'disk_min_file_size': 2 ** 15, # 32kb + 'disk_min_file_size': 2**15, # 32kb 'disk_pickle_protocol': pickle.HIGHEST_PROTOCOL, } @@ -212,7 +212,7 @@ def store(self, value, read, key=UNKNOWN): size = op.getsize(full_path) return size, MODE_TEXT, filename, None elif read: - reader = ft.partial(value.read, 2 ** 22) + reader = ft.partial(value.read, 2**22) filename, full_path = self.filename(key, value) iterator = iter(reader, b'') size = self._write(full_path, iterator, 'xb') diff --git a/diskcache/fanout.py b/diskcache/fanout.py index 8fe51d9..5283490 100644 --- a/diskcache/fanout.py +++ b/diskcache/fanout.py @@ -45,7 +45,7 @@ def __init__( timeout=timeout, disk=disk, size_limit=size_limit, - **settings + **settings, ) for num in range(shards) ) diff --git a/tests/benchmark_glob.py b/tests/benchmark_glob.py index 7f0bf7c..7da5fd3 100644 --- a/tests/benchmark_glob.py +++ b/tests/benchmark_glob.py @@ -22,7 +22,7 @@ print(template % ('Count', 'Time')) print(' '.join(['=' * size] * len(cols))) -for count in [10 ** exp for exp in range(6)]: +for count in [10**exp for exp in range(6)]: for value in range(count): with open(op.join('tmp', '%s.tmp' % value), 'wb') as writer: pass diff --git a/tests/settings.py b/tests/settings.py index 1a2f569..04aee85 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -25,7 +25,7 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True -ALLOWED_HOSTS = [u'testserver'] +ALLOWED_HOSTS = ['testserver'] # Application definition diff --git a/tests/stress_test_core.py b/tests/stress_test_core.py index c30fa3f..2b2578b 100644 --- a/tests/stress_test_core.py +++ b/tests/stress_test_core.py @@ -33,16 +33,14 @@ def make_long(): def make_unicode(): word_size = random.randint(1, 26) - word = u''.join( - random.sample(u'abcdefghijklmnopqrstuvwxyz', word_size) - ) + word = ''.join(random.sample('abcdefghijklmnopqrstuvwxyz', word_size)) size = random.randint(1, int(200 / 13)) return word * size def make_bytes(): word_size = random.randint(1, 26) - word = u''.join( - random.sample(u'abcdefghijklmnopqrstuvwxyz', word_size) + word = ''.join( + random.sample('abcdefghijklmnopqrstuvwxyz', word_size) ).encode('utf-8') size = random.randint(1, int(200 / 13)) return word * size @@ -77,18 +75,16 @@ def make_long(): def make_unicode(): word_size = random.randint(1, 26) - word = u''.join( - random.sample(u'abcdefghijklmnopqrstuvwxyz', word_size) - ) - size = random.randint(1, int(2 ** 16 / 13)) + word = ''.join(random.sample('abcdefghijklmnopqrstuvwxyz', word_size)) + size = random.randint(1, int(2**16 / 13)) return word * size def make_bytes(): word_size = random.randint(1, 26) - word = u''.join( - random.sample(u'abcdefghijklmnopqrstuvwxyz', word_size) + word = ''.join( + random.sample('abcdefghijklmnopqrstuvwxyz', word_size) ).encode('utf-8') - size = random.randint(1, int(2 ** 16 / 13)) + size = random.randint(1, int(2**16 / 13)) return word * size def make_float(): @@ -233,7 +229,7 @@ def percentile(sequence, percent): def stress_test( create=True, delete=True, - eviction_policy=u'least-recently-stored', + eviction_policy='least-recently-stored', processes=1, threads=1, ): @@ -293,17 +289,17 @@ def stress_test( def stress_test_lru(): """Stress test least-recently-used eviction policy.""" - stress_test(eviction_policy=u'least-recently-used') + stress_test(eviction_policy='least-recently-used') def stress_test_lfu(): """Stress test least-frequently-used eviction policy.""" - stress_test(eviction_policy=u'least-frequently-used') + stress_test(eviction_policy='least-frequently-used') def stress_test_none(): """Stress test 'none' eviction policy.""" - stress_test(eviction_policy=u'none') + stress_test(eviction_policy='none') def stress_test_mp(): @@ -396,7 +392,7 @@ def stress_test_mp(): '-v', '--eviction-policy', type=str, - default=u'least-recently-stored', + default='least-recently-stored', ) args = parser.parse_args() diff --git a/tests/stress_test_fanout.py b/tests/stress_test_fanout.py index d3b67e3..e78dda5 100644 --- a/tests/stress_test_fanout.py +++ b/tests/stress_test_fanout.py @@ -32,16 +32,14 @@ def make_long(): def make_unicode(): word_size = random.randint(1, 26) - word = u''.join( - random.sample(u'abcdefghijklmnopqrstuvwxyz', word_size) - ) + word = ''.join(random.sample('abcdefghijklmnopqrstuvwxyz', word_size)) size = random.randint(1, int(200 / 13)) return word * size def make_bytes(): word_size = random.randint(1, 26) - word = u''.join( - random.sample(u'abcdefghijklmnopqrstuvwxyz', word_size) + word = ''.join( + random.sample('abcdefghijklmnopqrstuvwxyz', word_size) ).encode('utf-8') size = random.randint(1, int(200 / 13)) return word * size @@ -76,18 +74,16 @@ def make_long(): def make_unicode(): word_size = random.randint(1, 26) - word = u''.join( - random.sample(u'abcdefghijklmnopqrstuvwxyz', word_size) - ) - size = random.randint(1, int(2 ** 16 / 13)) + word = ''.join(random.sample('abcdefghijklmnopqrstuvwxyz', word_size)) + size = random.randint(1, int(2**16 / 13)) return word * size def make_bytes(): word_size = random.randint(1, 26) - word = u''.join( - random.sample(u'abcdefghijklmnopqrstuvwxyz', word_size) + word = ''.join( + random.sample('abcdefghijklmnopqrstuvwxyz', word_size) ).encode('utf-8') - size = random.randint(1, int(2 ** 16 / 13)) + size = random.randint(1, int(2**16 / 13)) return word * size def make_float(): @@ -224,7 +220,7 @@ def percentile(sequence, percent): def stress_test( create=True, delete=True, - eviction_policy=u'least-recently-stored', + eviction_policy='least-recently-stored', processes=1, threads=1, ): @@ -284,17 +280,17 @@ def stress_test( def stress_test_lru(): """Stress test least-recently-used eviction policy.""" - stress_test(eviction_policy=u'least-recently-used') + stress_test(eviction_policy='least-recently-used') def stress_test_lfu(): """Stress test least-frequently-used eviction policy.""" - stress_test(eviction_policy=u'least-frequently-used') + stress_test(eviction_policy='least-frequently-used') def stress_test_none(): """Stress test 'none' eviction policy.""" - stress_test(eviction_policy=u'none') + stress_test(eviction_policy='none') def stress_test_mp(): @@ -387,7 +383,7 @@ def stress_test_mp(): '-v', '--eviction-policy', type=str, - default=u'least-recently-stored', + default='least-recently-stored', ) args = parser.parse_args() diff --git a/tests/test_core.py b/tests/test_core.py index 55ca962..356d104 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -38,11 +38,11 @@ def test_init(cache): def test_init_disk(): - with dc.Cache(disk_pickle_protocol=1, disk_min_file_size=2 ** 20) as cache: + with dc.Cache(disk_pickle_protocol=1, disk_min_file_size=2**20) as cache: key = (None, 0, 'abc') cache[key] = 0 cache.check() - assert cache.disk_min_file_size == 2 ** 20 + assert cache.disk_min_file_size == 2**20 assert cache.disk_pickle_protocol == 1 shutil.rmtree(cache.directory, ignore_errors=True) @@ -59,15 +59,15 @@ def test_disk_reset(): assert cache._disk.min_file_size == 0 assert cache._disk.pickle_protocol == 0 - cache.reset('disk_min_file_size', 2 ** 10) + cache.reset('disk_min_file_size', 2**10) cache.reset('disk_pickle_protocol', 2) cache[1] = value cache.check() - assert cache.disk_min_file_size == 2 ** 10 + assert cache.disk_min_file_size == 2**10 assert cache.disk_pickle_protocol == 2 - assert cache._disk.min_file_size == 2 ** 10 + assert cache._disk.min_file_size == 2**10 assert cache._disk.pickle_protocol == 2 shutil.rmtree(cache.directory, ignore_errors=True) @@ -150,7 +150,7 @@ def test_pragma_error(cache): cursor.fetchall = fetchall fetchall.side_effect = [sqlite3.OperationalError] * 60000 - size = 2 ** 28 + size = 2**28 with mock.patch('time.sleep', lambda num: 0): with mock.patch.object(cache, '_local', local): @@ -177,15 +177,15 @@ def __getattr__(self, name): def test_getsetdel(cache): values = [ (None, False), - ((None,) * 2 ** 20, False), + ((None,) * 2**20, False), (1234, False), - (2 ** 512, False), + (2**512, False), (56.78, False), - (u'hello', False), - (u'hello' * 2 ** 20, False), + ('hello', False), + ('hello' * 2**20, False), (b'world', False), - (b'world' * 2 ** 20, False), - (io.BytesIO(b'world' * 2 ** 20), True), + (b'world' * 2**20, False), + (io.BytesIO(b'world' * 2**20), True), ] for key, (value, file_like) in enumerate(values): @@ -229,7 +229,7 @@ def test_get_keyerror4(cache): func = mock.Mock(side_effect=IOError(errno.ENOENT, '')) cache.reset('statistics', True) - cache[0] = b'abcd' * 2 ** 20 + cache[0] = b'abcd' * 2**20 with mock.patch('diskcache.core.open', func): with pytest.raises((IOError, KeyError, OSError)): @@ -237,7 +237,7 @@ def test_get_keyerror4(cache): def test_read(cache): - cache.set(0, b'abcd' * 2 ** 20) + cache.set(0, b'abcd' * 2**20) with cache.read(0) as reader: assert reader is not None @@ -249,7 +249,7 @@ def test_read_keyerror(cache): def test_set_twice(cache): - large_value = b'abcd' * 2 ** 20 + large_value = b'abcd' * 2**20 cache[0] = 0 cache[0] = 1 @@ -283,7 +283,7 @@ def test_set_timeout(cache): with pytest.raises(dc.Timeout): try: with mock.patch.object(cache, '_local', local): - cache.set('a', 'b' * 2 ** 20) + cache.set('a', 'b' * 2**20) finally: cache.check() @@ -299,11 +299,11 @@ def test_get(cache): assert cache.get(2, {}) == {} assert cache.get(0, expire_time=True, tag=True) == (None, None, None) - assert cache.set(0, 0, expire=None, tag=u'number') + assert cache.set(0, 0, expire=None, tag='number') assert cache.get(0, expire_time=True) == (0, None) - assert cache.get(0, tag=True) == (0, u'number') - assert cache.get(0, expire_time=True, tag=True) == (0, None, u'number') + assert cache.get(0, tag=True) == (0, 'number') + assert cache.get(0, expire_time=True, tag=True) == (0, None, 'number') def test_get_expired_fast_path(cache): @@ -359,8 +359,8 @@ def test_pop(cache): assert cache.set('delta', 210) assert cache.pop('delta', expire_time=True) == (210, None) - assert cache.set('epsilon', '0' * 2 ** 20) - assert cache.pop('epsilon') == '0' * 2 ** 20 + assert cache.set('epsilon', '0' * 2**20) + assert cache.pop('epsilon') == '0' * 2**20 def test_pop_ioerror(cache): @@ -426,11 +426,11 @@ def test_stats(cache): def test_path(cache): - cache[0] = u'abc' - large_value = b'abc' * 2 ** 20 + cache[0] = 'abc' + large_value = b'abc' * 2**20 cache[1] = large_value - assert cache.get(0, read=True) == u'abc' + assert cache.get(0, read=True) == 'abc' with cache.get(1, read=True) as reader: assert reader.name is not None @@ -465,7 +465,7 @@ def test_expire_rows(cache): def test_least_recently_stored(cache): - cache.reset('eviction_policy', u'least-recently-stored') + cache.reset('eviction_policy', 'least-recently-stored') cache.reset('size_limit', int(10.1e6)) cache.reset('cull_limit', 2) @@ -500,7 +500,7 @@ def test_least_recently_stored(cache): def test_least_recently_used(cache): - cache.reset('eviction_policy', u'least-recently-used') + cache.reset('eviction_policy', 'least-recently-used') cache.reset('size_limit', int(10.1e6)) cache.reset('cull_limit', 5) @@ -530,7 +530,7 @@ def test_least_recently_used(cache): def test_least_frequently_used(cache): - cache.reset('eviction_policy', u'least-frequently-used') + cache.reset('eviction_policy', 'least-frequently-used') cache.reset('size_limit', int(10.1e6)) cache.reset('cull_limit', 5) @@ -558,8 +558,8 @@ def test_least_frequently_used(cache): def test_check(cache): - blob = b'a' * 2 ** 20 - keys = (0, 1, 1234, 56.78, u'hello', b'world', None) + blob = b'a' * 2**20 + keys = (0, 1, 1234, 56.78, 'hello', b'world', None) for key in keys: cache[key] = blob @@ -662,12 +662,12 @@ def test_clear_timeout(cache): def test_tag(cache): - assert cache.set(0, None, tag=u'zero') + assert cache.set(0, None, tag='zero') assert cache.set(1, None, tag=1234) assert cache.set(2, None, tag=5.67) assert cache.set(3, None, tag=b'three') - assert cache.get(0, tag=True) == (None, u'zero') + assert cache.get(0, tag=True) == (None, 'zero') assert cache.get(1, tag=True) == (None, 1234) assert cache.get(2, tag=True) == (None, 5.67) assert cache.get(3, tag=True) == (None, b'three') @@ -675,11 +675,11 @@ def test_tag(cache): def test_with(cache): with dc.Cache(cache.directory) as tmp: - tmp[u'a'] = 0 - tmp[u'b'] = 1 + tmp['a'] = 0 + tmp['b'] = 1 - assert cache[u'a'] == 0 - assert cache[u'b'] == 1 + assert cache['a'] == 0 + assert cache['b'] == 1 def test_contains(cache): @@ -708,7 +708,7 @@ def test_add(cache): def test_add_large_value(cache): - value = b'abcd' * 2 ** 20 + value = b'abcd' * 2**20 assert cache.add(b'test-key', value) assert cache.get(b'test-key') == value assert not cache.add(b'test-key', value * 2) @@ -919,7 +919,7 @@ def test_push_peek_expire(cache): def test_push_pull_large_value(cache): - value = b'test' * (2 ** 20) + value = b'test' * (2**20) cache.push(value) assert cache.pull() == (500000000000000, value) assert len(cache) == 0 @@ -927,7 +927,7 @@ def test_push_pull_large_value(cache): def test_push_peek_large_value(cache): - value = b'test' * (2 ** 20) + value = b'test' * (2**20) cache.push(value) assert cache.peek() == (500000000000000, value) assert len(cache) == 1 @@ -1144,8 +1144,8 @@ def test_cull_timeout(cache): def test_key_roundtrip(cache): - key_part_0 = u'part0' - key_part_1 = u'part1' + key_part_0 = 'part0' + key_part_1 = 'part1' to_test = [ (key_part_0, key_part_1), [key_part_0, key_part_1], @@ -1354,7 +1354,7 @@ def foo(*args, **kwargs): def test_cleanup_dirs(cache): - value = b'\0' * 2 ** 20 + value = b'\0' * 2**20 start_count = len(os.listdir(cache.directory)) for i in range(10): cache[i] = value @@ -1370,7 +1370,7 @@ def test_disk_write_os_error(cache): func = mock.Mock(side_effect=[OSError] * 10) with mock.patch('diskcache.core.open', func): with pytest.raises(OSError): - cache[0] = '\0' * 2 ** 20 + cache[0] = '\0' * 2**20 def test_memoize_ignore(cache): diff --git a/tests/test_djangocache.py b/tests/test_djangocache.py index 5f83b81..b5cc2a8 100644 --- a/tests/test_djangocache.py +++ b/tests/test_djangocache.py @@ -1033,7 +1033,7 @@ def test_directory(self): self.assertTrue('tmp' in cache.directory) def test_read(self): - value = b'abcd' * 2 ** 20 + value = b'abcd' * 2**20 result = cache.set(b'test-key', value) self.assertTrue(result) diff --git a/tests/test_fanout.py b/tests/test_fanout.py index f212fac..deea03f 100644 --- a/tests/test_fanout.py +++ b/tests/test_fanout.py @@ -34,7 +34,7 @@ def test_init(cache): del default_settings['size_limit'] for key, value in default_settings.items(): assert getattr(cache, key) == value - assert cache.size_limit == 2 ** 27 + assert cache.size_limit == 2**27 cache.check() @@ -229,15 +229,15 @@ def test_incr_concurrent(): def test_getsetdel(cache): values = [ (None, False), - ((None,) * 2 ** 10, False), + ((None,) * 2**10, False), (1234, False), - (2 ** 512, False), + (2**512, False), (56.78, False), - (u'hello', False), - (u'hello' * 2 ** 10, False), + ('hello', False), + ('hello' * 2**10, False), (b'world', False), - (b'world' * 2 ** 10, False), - (io.BytesIO(b'world' * 2 ** 10), True), + (b'world' * 2**10, False), + (io.BytesIO(b'world' * 2**10), True), ] for key, (value, file_like) in enumerate(values): @@ -341,7 +341,7 @@ def test_tag_index(cache): def test_read(cache): - cache.set(0, b'abcd' * 2 ** 20) + cache.set(0, b'abcd' * 2**20) with cache.read(0) as reader: assert reader is not None From a53283d554fbe7bd678067f24d6f45d57de3f83b Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 10 Apr 2023 23:25:16 -0700 Subject: [PATCH 08/31] Update requirements --- requirements-dev.txt | 23 +++++++++++++++++++++++ requirements.txt | 22 ---------------------- 2 files changed, 23 insertions(+), 22 deletions(-) create mode 100644 requirements-dev.txt diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..6149361 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,23 @@ +-e . +blue +coverage +django==4.2.* +django_redis +doc8 +flake8 +ipython +jedi +pickleDB +pylibmc +pylint +pytest +pytest-cov +pytest-django +pytest-env +pytest-xdist +rstcheck +sphinx +sqlitedict +tox +twine +wheel diff --git a/requirements.txt b/requirements.txt index efb2160..d6e1198 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,23 +1 @@ -e . -blue -coverage -django==3.2.* -django_redis -doc8 -flake8 -ipython -jedi==0.17.* # Remove after IPython bug fixed. -pickleDB -pylibmc -pylint -pytest -pytest-cov -pytest-django -pytest-env -pytest-xdist -rstcheck -sphinx -sqlitedict -tox -twine -wheel From b22a7d58c3dbf3f71fa4f9156ccbd4892ebf3fe2 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 10 Apr 2023 23:25:35 -0700 Subject: [PATCH 09/31] Update pylint --- .pylintrc | 829 ++++++++++++++++++++++++---------------------- diskcache/core.py | 6 +- 2 files changed, 439 insertions(+), 396 deletions(-) diff --git a/.pylintrc b/.pylintrc index 6baa978..dc1490a 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,27 +1,77 @@ -[MASTER] +[MAIN] + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Clear in-memory caches upon conclusion of linting. Useful if running pylint +# in a server-like mode. +clear-cache-post-run=no + +# Load and enable all available extensions. Use --list-extensions to see a list +# all available extensions. +#enable-all-extensions= + +# In error mode, messages with a category besides ERROR or FATAL are +# suppressed, and no reports are done by default. Error mode is compatible with +# disabling specific errors. +#errors-only= + +# Always return a 0 (non-error) status code, even if lint errors are found. +# This is primarily useful in continuous integration scripts. +#exit-zero= # A comma-separated list of package or module names from where C extensions may # be loaded. Extensions are loading into the active Python interpreter and may # run arbitrary code. +extension-pkg-allow-list= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. (This is an alternative name to extension-pkg-allow-list +# for backward compatibility.) extension-pkg-whitelist= -# Specify a score threshold to be exceeded before program exits with error. -fail-under=10.0 +# Return non-zero exit code if any of these messages/categories are detected, +# even if score is above --fail-under value. Syntax same as enable. Messages +# specified are enabled, while categories only check already-enabled messages. +fail-on= -# Add files or directories to the blacklist. They should be base names, not -# paths. +# Specify a score threshold under which the program will exit with error. +fail-under=10 + +# Interpret the stdin as a python script, whose filename needs to be passed as +# the module_or_package argument. +#from-stdin= + +# Files or directories to be skipped. They should be base names, not paths. ignore=CVS -# Add files or directories matching the regex patterns to the blacklist. The -# regex matches against base names, not paths. -ignore-patterns= +# Add files or directories matching the regular expressions patterns to the +# ignore-list. The regex matches against paths and can be in Posix or Windows +# format. Because '\\' represents the directory delimiter on Windows systems, +# it can't be used as an escape character. +ignore-paths= + +# Files or directories matching the regular expression patterns are skipped. +# The regex matches against base names, not paths. The default value ignores +# Emacs file locks +ignore-patterns=^\.# + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis). It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= # Python code to execute, usually for sys.path manipulation such as # pygtk.require(). #init-hook= # Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the -# number of processors available to use. +# number of processors available to use, and will cap the count on Windows to +# avoid hangs. jobs=1 # Control the amount of potential inferred values when inferring a single @@ -36,6 +86,19 @@ load-plugins= # Pickle collected data for later comparisons. persistent=yes +# Minimum Python version to use for version dependent checks. Will default to +# the version used to run pylint. +py-version=3.11 + +# Discover python modules and packages in the file system subtree. +recursive=no + +# Add paths to the list of the source roots. Supports globbing patterns. The +# source root is an absolute path or a path relative to the current working +# directory used to determine a package namespace for modules located under the +# source root. +source-roots= + # When enabled, pylint would attempt to guess common misconfiguration and emit # user-friendly hints instead of false-positive error messages. suggestion-mode=yes @@ -44,321 +107,8 @@ suggestion-mode=yes # active Python interpreter and may run arbitrary code. unsafe-load-any-extension=no - -[MESSAGES CONTROL] - -# Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED. -confidence= - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once). You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use "--disable=all --enable=classes -# --disable=W". -disable=print-statement, - parameter-unpacking, - unpacking-in-except, - old-raise-syntax, - backtick, - long-suffix, - old-ne-operator, - old-octal-literal, - import-star-module-level, - non-ascii-bytes-literal, - raw-checker-failed, - bad-inline-option, - locally-disabled, - file-ignored, - suppressed-message, - useless-suppression, - deprecated-pragma, - use-symbolic-message-instead, - apply-builtin, - basestring-builtin, - buffer-builtin, - cmp-builtin, - coerce-builtin, - execfile-builtin, - file-builtin, - long-builtin, - raw_input-builtin, - reduce-builtin, - standarderror-builtin, - unicode-builtin, - xrange-builtin, - coerce-method, - delslice-method, - getslice-method, - setslice-method, - no-absolute-import, - old-division, - dict-iter-method, - dict-view-method, - next-method-called, - metaclass-assignment, - indexing-exception, - raising-string, - reload-builtin, - oct-method, - hex-method, - nonzero-method, - cmp-method, - input-builtin, - round-builtin, - intern-builtin, - unichr-builtin, - map-builtin-not-iterating, - zip-builtin-not-iterating, - range-builtin-not-iterating, - filter-builtin-not-iterating, - using-cmp-argument, - eq-without-hash, - div-method, - idiv-method, - rdiv-method, - exception-message-attribute, - invalid-str-codec, - sys-max-int, - bad-python3-import, - deprecated-string-function, - deprecated-str-translate-call, - deprecated-itertools-function, - deprecated-types-field, - next-method-defined, - dict-items-not-iterating, - dict-keys-not-iterating, - dict-values-not-iterating, - deprecated-operator-function, - deprecated-urllib-function, - xreadlines-attribute, - deprecated-sys-function, - exception-escape, - comprehension-escape, - no-member, - no-else-return, - duplicate-code, - inconsistent-return-statements, - consider-using-f-string, - -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time (only on the command line, not in the configuration file where -# it should appear only once). See also the "--disable" option for examples. -enable=c-extension-no-member - - -[REPORTS] - -# Python expression which should return a score less than or equal to 10. You -# have access to the variables 'error', 'warning', 'refactor', and 'convention' -# which contain the number of messages in each category, as well as 'statement' -# which is the total number of statements analyzed. This score is used by the -# global evaluation report (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) - -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details. -#msg-template= - -# Set the output format. Available formats are text, parseable, colorized, json -# and msvs (visual studio). You can also give a reporter class, e.g. -# mypackage.mymodule.MyReporterClass. -output-format=text - -# Tells whether to display a full report or only the messages. -reports=no - -# Activate the evaluation score. -score=yes - - -[REFACTORING] - -# Maximum number of nested blocks for function / method body -max-nested-blocks=5 - -# Complete name of functions that never returns. When checking for -# inconsistent-return-statements if a never returning function is called then -# it will be considered as an explicit return statement and no message will be -# printed. -never-returning-functions=sys.exit - - -[LOGGING] - -# The type of string formatting that logging methods do. `old` means using % -# formatting, `new` is for `{}` formatting. -logging-format-style=old - -# Logging modules to check that the string format arguments are in logging -# function parameter format. -logging-modules=logging - - -[SPELLING] - -# Limits count of emitted suggestions for spelling mistakes. -max-spelling-suggestions=4 - -# Spelling dictionary name. Available dictionaries: none. To make it work, -# install the python-enchant package. -spelling-dict= - -# List of comma separated words that should not be checked. -spelling-ignore-words= - -# A path to a file that contains the private dictionary; one word per line. -spelling-private-dict-file= - -# Tells whether to store unknown words to the private dictionary (see the -# --spelling-private-dict-file option) instead of raising a message. -spelling-store-unknown-words=no - - -[MISCELLANEOUS] - -# List of note tags to take in consideration, separated by a comma. -notes=FIXME, - XXX, - TODO - -# Regular expression of note tags to take in consideration. -#notes-rgx= - - -[TYPECHECK] - -# List of decorators that produce context managers, such as -# contextlib.contextmanager. Add to this list to register other decorators that -# produce valid context managers. -contextmanager-decorators=contextlib.contextmanager - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. -generated-members= - -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes - -# Tells whether to warn about missing members when the owner of the attribute -# is inferred to be None. -ignore-none=yes - -# This flag controls whether pylint should warn about no-member and similar -# checks whenever an opaque object is returned when inferring. The inference -# can return multiple potential results while evaluating a Python object, but -# some branches might not be evaluated, which results in partial inference. In -# that case, it might be useful to still emit no-member and other checks for -# the rest of the inferred objects. -ignore-on-opaque-inference=yes - -# List of class names for which member attributes should not be checked (useful -# for classes with dynamically set attributes). This supports the use of -# qualified names. -ignored-classes=optparse.Values,thread._local,_thread._local - -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis). It -# supports qualified module names, as well as Unix pattern matching. -ignored-modules= - -# Show a hint with possible names when a member name was not found. The aspect -# of finding the hint is based on edit distance. -missing-member-hint=yes - -# The minimum edit distance a name should have in order to be considered a -# similar match for a missing member name. -missing-member-hint-distance=1 - -# The total number of similar names that should be taken in consideration when -# showing a hint for a missing member. -missing-member-max-choices=1 - -# List of decorators that change the signature of a decorated function. -signature-mutators= - - -[VARIABLES] - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid defining new builtins when possible. -additional-builtins= - -# Tells whether unused global variables should be treated as a violation. -allow-global-unused-variables=yes - -# List of strings which can identify a callback function by name. A callback -# name must start or end with one of those strings. -callbacks=cb_, - _cb - -# A regular expression matching the name of dummy variables (i.e. expected to -# not be used). -dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ - -# Argument names that match this expression will be ignored. Default to name -# with leading underscore. -ignored-argument-names=_.*|^ignored_|^unused_ - -# Tells whether we should check for unused import in __init__ files. -init-import=no - -# List of qualified module names which can have objects that can redefine -# builtins. -redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io - - -[FORMAT] - -# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. -expected-line-ending-format= - -# Regexp for a line that is allowed to be longer than the limit. -ignore-long-lines=^\s*(# )??$ - -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren=4 - -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string=' ' - -# Maximum number of characters on a single line. -max-line-length=100 - -# Maximum number of lines in a module. -max-module-lines=3000 - -# Allow the body of a class to be on the same line as the declaration if body -# contains single statement. -single-line-class-stmt=no - -# Allow the body of an if to be on the same line as the test if there is no -# else. -single-line-if-stmt=no - - -[SIMILARITIES] - -# Ignore comments when computing similarities. -ignore-comments=yes - -# Ignore docstrings when computing similarities. -ignore-docstrings=yes - -# Ignore imports when computing similarities. -ignore-imports=yes - -# Minimum lines number of a similarity. -min-similarity-lines=4 +# In verbose mode, extra non-checker-related info will be displayed. +#verbose= [BASIC] @@ -367,13 +117,15 @@ min-similarity-lines=4 argument-naming-style=snake_case # Regular expression matching correct argument names. Overrides argument- -# naming-style. +# naming-style. If left empty, argument names will be checked with the set +# naming style. #argument-rgx= # Naming style matching correct attribute names. attr-naming-style=snake_case # Regular expression matching correct attribute names. Overrides attr-naming- +# style. If left empty, attribute names will be checked with the set naming # style. #attr-rgx= @@ -393,20 +145,30 @@ bad-names-rgxs= class-attribute-naming-style=any # Regular expression matching correct class attribute names. Overrides class- -# attribute-naming-style. +# attribute-naming-style. If left empty, class attribute names will be checked +# with the set naming style. #class-attribute-rgx= +# Naming style matching correct class constant names. +class-const-naming-style=UPPER_CASE + +# Regular expression matching correct class constant names. Overrides class- +# const-naming-style. If left empty, class constant names will be checked with +# the set naming style. +#class-const-rgx= + # Naming style matching correct class names. class-naming-style=PascalCase # Regular expression matching correct class names. Overrides class-naming- -# style. +# style. If left empty, class names will be checked with the set naming style. #class-rgx= # Naming style matching correct constant names. const-naming-style=UPPER_CASE # Regular expression matching correct constant names. Overrides const-naming- +# style. If left empty, constant names will be checked with the set naming # style. #const-rgx= @@ -418,7 +180,8 @@ docstring-min-length=-1 function-naming-style=snake_case # Regular expression matching correct function names. Overrides function- -# naming-style. +# naming-style. If left empty, function names will be checked with the set +# naming style. #function-rgx= # Good variable names which should always be accepted, separated by a comma. @@ -440,21 +203,22 @@ include-naming-hint=no inlinevar-naming-style=any # Regular expression matching correct inline iteration names. Overrides -# inlinevar-naming-style. +# inlinevar-naming-style. If left empty, inline iteration names will be checked +# with the set naming style. #inlinevar-rgx= # Naming style matching correct method names. method-naming-style=snake_case # Regular expression matching correct method names. Overrides method-naming- -# style. +# style. If left empty, method names will be checked with the set naming style. #method-rgx= # Naming style matching correct module names. module-naming-style=snake_case # Regular expression matching correct module names. Overrides module-naming- -# style. +# style. If left empty, module names will be checked with the set naming style. #module-rgx= # Colon-delimited sets of names that determine each other's naming style when @@ -470,90 +234,56 @@ no-docstring-rgx=^_ # These decorators are taken in consideration only for invalid-name. property-classes=abc.abstractproperty +# Regular expression matching correct type alias names. If left empty, type +# alias names will be checked with the set naming style. +#typealias-rgx= + +# Regular expression matching correct type variable names. If left empty, type +# variable names will be checked with the set naming style. +#typevar-rgx= + # Naming style matching correct variable names. variable-naming-style=snake_case # Regular expression matching correct variable names. Overrides variable- -# naming-style. +# naming-style. If left empty, variable names will be checked with the set +# naming style. #variable-rgx= -[STRING] - -# This flag controls whether inconsistent-quotes generates a warning when the -# character used as a quote delimiter is used inconsistently within a module. -check-quote-consistency=no +[CLASSES] -# This flag controls whether the implicit-str-concat should generate a warning -# on implicit string concatenation in sequences defined over several lines. -check-str-concat-over-line-jumps=no - - -[IMPORTS] - -# List of modules that can be imported at any level, not just the top level -# one. -allow-any-import-level= - -# Allow wildcard imports from modules that define __all__. -allow-wildcard-with-all=no - -# Analyse import fallback blocks. This can be used to support both Python 2 and -# 3 compatible code, which means that the block might have code that exists -# only in one or another interpreter, leading to false positives when analysed. -analyse-fallback-blocks=no - -# Deprecated modules which should not be used, separated by a comma. -deprecated-modules=optparse,tkinter.tix - -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled). -ext-import-graph= - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled). -import-graph= - -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled). -int-import-graph= - -# Force import order to recognize a module as part of the standard -# compatibility libraries. -known-standard-library= - -# Force import order to recognize a module as part of a third party library. -known-third-party=enchant - -# Couples of modules and preferred modules, separated by a comma. -preferred-modules= - - -[CLASSES] +# Warn about protected attribute access inside special methods +check-protected-access-in-special-methods=no # List of method names used to declare (i.e. assign) instance attributes. defining-attr-methods=__init__, __new__, setUp, + asyncSetUp, __post_init__ # List of member names, which should be excluded from the protected access # warning. -exclude-protected=_asdict, - _fields, - _replace, - _source, - _make +exclude-protected=_asdict,_fields,_replace,_source,_make,os._exit # List of valid names for the first argument in a class method. valid-classmethod-first-arg=cls # List of valid names for the first argument in a metaclass class method. -valid-metaclass-classmethod-first-arg=cls +valid-metaclass-classmethod-first-arg=mcs [DESIGN] +# List of regular expressions of class ancestor names to ignore when counting +# public methods (see R0903) +exclude-too-few-public-methods= + +# List of qualified class names to ignore when counting class parents (see +# R0901) +ignored-parents= + # Maximum number of arguments for function / method. max-args=8 @@ -587,7 +317,320 @@ min-public-methods=2 [EXCEPTIONS] -# Exceptions that will emit a warning when being caught. Defaults to -# "BaseException, Exception". -overgeneral-exceptions=BaseException, - Exception +# Exceptions that will emit a warning when caught. +overgeneral-exceptions=builtins.BaseException,builtins.Exception + + +[FORMAT] + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=100 + +# Maximum number of lines in a module. +max-module-lines=2500 + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[IMPORTS] + +# List of modules that can be imported at any level, not just the top level +# one. +allow-any-import-level= + +# Allow explicit reexports by alias from a package __init__. +allow-reexport-from-package=no + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Deprecated modules which should not be used, separated by a comma. +deprecated-modules= + +# Output a graph (.gv or any supported image format) of external dependencies +# to the given file (report RP0402 must not be disabled). +ext-import-graph= + +# Output a graph (.gv or any supported image format) of all (i.e. internal and +# external) dependencies to the given file (report RP0402 must not be +# disabled). +import-graph= + +# Output a graph (.gv or any supported image format) of internal dependencies +# to the given file (report RP0402 must not be disabled). +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Couples of modules and preferred modules, separated by a comma. +preferred-modules= + + +[LOGGING] + +# The type of string formatting that logging methods do. `old` means using % +# formatting, `new` is for `{}` formatting. +logging-format-style=old + +# Logging modules to check that the string format arguments are in logging +# function parameter format. +logging-modules=logging + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, +# UNDEFINED. +confidence=HIGH, + CONTROL_FLOW, + INFERENCE, + INFERENCE_FAILURE, + UNDEFINED + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then re-enable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable=raw-checker-failed, + bad-inline-option, + locally-disabled, + file-ignored, + suppressed-message, + useless-suppression, + deprecated-pragma, + use-symbolic-message-instead, + consider-using-f-string, + no-member, + no-else-return, + no-else-raise, + inconsistent-return-statements + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable=c-extension-no-member + + +[METHOD_ARGS] + +# List of qualified names (i.e., library.method) which require a timeout +# parameter e.g. 'requests.api.get,requests.api.post' +timeout-methods=requests.api.delete,requests.api.get,requests.api.head,requests.api.options,requests.api.patch,requests.api.post,requests.api.put,requests.api.request + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME, + XXX, + TODO + +# Regular expression of note tags to take in consideration. +notes-rgx= + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=sys.exit,argparse.parse_error + + +[REPORTS] + +# Python expression which should return a score less than or equal to 10. You +# have access to the variables 'fatal', 'error', 'warning', 'refactor', +# 'convention', and 'info' which contain the number of messages in each +# category, as well as 'statement' which is the total number of statements +# analyzed. This score is used by the global evaluation report (RP0004). +evaluation=max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +msg-template= + +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +#output-format= + +# Tells whether to display a full report or only the messages. +reports=no + +# Activate the evaluation score. +score=yes + + +[SIMILARITIES] + +# Comments are removed from the similarity computation +ignore-comments=yes + +# Docstrings are removed from the similarity computation +ignore-docstrings=yes + +# Imports are removed from the similarity computation +ignore-imports=yes + +# Signatures are removed from the similarity computation +ignore-signatures=yes + +# Minimum lines number of a similarity. +min-similarity-lines=20 + + +[SPELLING] + +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions=4 + +# Spelling dictionary name. No available dictionaries : You need to install +# both the python package and the system dependency for enchant to work.. +spelling-dict= + +# List of comma separated words that should be considered directives if they +# appear at the beginning of a comment and should not be checked. +spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy: + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains the private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to the private dictionary (see the +# --spelling-private-dict-file option) instead of raising a message. +spelling-store-unknown-words=no + + +[STRING] + +# This flag controls whether inconsistent-quotes generates a warning when the +# character used as a quote delimiter is used inconsistently within a module. +check-quote-consistency=no + +# This flag controls whether the implicit-str-concat should generate a warning +# on implicit string concatenation in sequences defined over several lines. +check-str-concat-over-line-jumps=no + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# Tells whether to warn about missing members when the owner of the attribute +# is inferred to be None. +ignore-none=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes + +# List of symbolic message names to ignore for Mixin members. +ignored-checks-for-mixins=no-member, + not-async-context-manager, + not-context-manager, + attribute-defined-outside-init + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local,argparse.Namespace + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + +# Regex pattern to define which classes are considered mixins. +mixin-class-rgx=.*[Mm]ixin + +# List of decorators that change the signature of a decorated function. +signature-mutators= + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid defining new builtins when possible. +additional-builtins= + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes + +# List of names allowed to shadow builtins +allowed-redefined-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_, + _cb + +# A regular expression matching the name of dummy variables (i.e. expected to +# not be used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io diff --git a/diskcache/core.py b/diskcache/core.py index 05a0854..9f3a597 100644 --- a/diskcache/core.py +++ b/diskcache/core.py @@ -170,7 +170,7 @@ def get(self, key, raw): :return: corresponding Python key """ - # pylint: disable=no-self-use,unidiomatic-typecheck + # pylint: disable=unidiomatic-typecheck if raw: return bytes(key) if type(key) is sqlite3.Binary else key else: @@ -228,7 +228,6 @@ def store(self, value, read, key=UNKNOWN): return len(result), MODE_PICKLE, filename, None def _write(self, full_path, iterator, mode, encoding=None): - # pylint: disable=no-self-use full_dir, _ = op.split(full_path) for count in range(1, 11): @@ -264,7 +263,7 @@ def fetch(self, mode, filename, value, read): :raises: IOError if the value cannot be read """ - # pylint: disable=no-self-use,unidiomatic-typecheck,consider-using-with + # pylint: disable=unidiomatic-typecheck,consider-using-with if mode == MODE_RAW: return bytes(value) if type(value) is sqlite3.Binary else value elif mode == MODE_BINARY: @@ -1378,6 +1377,7 @@ def delete(self, key, retry=False): :raises Timeout: if database timeout occurs """ + # pylint: disable=unnecessary-dunder-call try: return self.__delitem__(key, retry=retry) except KeyError: From 471aa5e551aed186dd34d5bc7d345be835efd6f3 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 10 Apr 2023 23:26:38 -0700 Subject: [PATCH 10/31] Drop Python 3.7 from testing --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 3735ebc..e7217a7 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist=bluecheck,doc8,docs,isortcheck,flake8,mypy,pylint,rstcheck,py37,py38,py39,py310,py311 +envlist=bluecheck,doc8,docs,isortcheck,flake8,mypy,pylint,rstcheck,py38,py39,py310,py311 skip_missing_interpreters=True [testenv] From c14345f105f14eba45986b09ec96d87b8997c9cc Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 10 Apr 2023 23:42:38 -0700 Subject: [PATCH 11/31] Update tests for Django 4.2 --- tests/test_djangocache.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_djangocache.py b/tests/test_djangocache.py index b5cc2a8..734ba1b 100644 --- a/tests/test_djangocache.py +++ b/tests/test_djangocache.py @@ -870,7 +870,6 @@ def test_custom_key_func(self): def test_cache_write_unpicklable_object(self): fetch_middleware = FetchFromCacheMiddleware(empty_response) - fetch_middleware.cache = cache request = self.factory.get('/cache/test') request._cache_update_cache = True @@ -887,7 +886,6 @@ def get_response(req): return response update_middleware = UpdateCacheMiddleware(get_response) - update_middleware.cache = cache response = update_middleware(request) get_cache_data = fetch_middleware.process_request(request) From 0294d58cd0cb72dd6affbbc46b3ff05d96d015cc Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 10 Apr 2023 23:43:15 -0700 Subject: [PATCH 12/31] Bump version to v5.5.0 --- diskcache/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/diskcache/__init__.py b/diskcache/__init__.py index f7aa771..8d7e28c 100644 --- a/diskcache/__init__.py +++ b/diskcache/__init__.py @@ -61,8 +61,8 @@ pass __title__ = 'diskcache' -__version__ = '5.4.0' -__build__ = 0x050400 +__version__ = '5.5.0' +__build__ = 0x050500 __author__ = 'Grant Jenks' __license__ = 'Apache 2.0' __copyright__ = 'Copyright 2016-2023 Grant Jenks' From 6cd6888a16be1d531a19ca91d5e0daa6edac9718 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 10 Apr 2023 23:45:14 -0700 Subject: [PATCH 13/31] Drop 3.7 from CI --- .github/workflows/integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 2d83ad3..07aceec 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -32,7 +32,7 @@ jobs: max-parallel: 8 matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: [3.7, 3.8, 3.9, '3.10', 3.11] + python-version: [3.8, 3.9, '3.10', 3.11] steps: - name: Set up Python ${{ matrix.python-version }} x64 From bbac13b54f9ac0a5c45b7cfbd242869b41042cac Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 10 Apr 2023 23:52:57 -0700 Subject: [PATCH 14/31] Install dev requirements for wheel package --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 676593c..21b6e5b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -26,7 +26,7 @@ jobs: - name: Install dependencies run: | pip install --upgrade pip - pip install -r requirements.txt + pip install -r requirements-dev.txt - name: Create source dist run: python setup.py sdist From 0f5d8ed63406e26de96701f98aa585b4fb26f6dd Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 10 Apr 2023 23:53:20 -0700 Subject: [PATCH 15/31] Bump version to 5.5.1 --- diskcache/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/diskcache/__init__.py b/diskcache/__init__.py index 8d7e28c..95dafb3 100644 --- a/diskcache/__init__.py +++ b/diskcache/__init__.py @@ -61,8 +61,8 @@ pass __title__ = 'diskcache' -__version__ = '5.5.0' -__build__ = 0x050500 +__version__ = '5.5.1' +__build__ = 0x050501 __author__ = 'Grant Jenks' __license__ = 'Apache 2.0' __copyright__ = 'Copyright 2016-2023 Grant Jenks' From fe5ee43ac5df1847556f280c696ab921f0910be2 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Sun, 16 Apr 2023 21:42:13 -0700 Subject: [PATCH 16/31] Close the cache explicitly before deleting the reference --- diskcache/persistent.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/diskcache/persistent.py b/diskcache/persistent.py index c3d570b..cce3736 100644 --- a/diskcache/persistent.py +++ b/diskcache/persistent.py @@ -533,12 +533,13 @@ def reverse(self): # GrantJ 2019-03-22 Consider using an algorithm that swaps the values # at two keys. Like self._cache.swap(key1, key2, retry=True) The swap # method would exchange the values at two given keys. Then, using a - # forward iterator and a reverse iterator, the reversis method could + # forward iterator and a reverse iterator, the reverse method could # avoid making copies of the values. temp = Deque(iterable=reversed(self)) self.clear() self.extend(temp) directory = temp.directory + temp.close() del temp rmtree(directory) From 9380c784d9e2954611b2ea309f1c657602085f25 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Sun, 16 Apr 2023 23:02:24 -0700 Subject: [PATCH 17/31] Oops, close the cache, not the deque --- diskcache/persistent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/diskcache/persistent.py b/diskcache/persistent.py index cce3736..01cf4e3 100644 --- a/diskcache/persistent.py +++ b/diskcache/persistent.py @@ -539,7 +539,7 @@ def reverse(self): self.clear() self.extend(temp) directory = temp.directory - temp.close() + temp._cache.close() del temp rmtree(directory) From f5a17ff0959a4cc7147d45a4cc8d8eef8d7416b0 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 17 Apr 2023 19:00:34 -0700 Subject: [PATCH 18/31] Shutup pylint --- diskcache/persistent.py | 1 + 1 file changed, 1 insertion(+) diff --git a/diskcache/persistent.py b/diskcache/persistent.py index 01cf4e3..c3f22b5 100644 --- a/diskcache/persistent.py +++ b/diskcache/persistent.py @@ -530,6 +530,7 @@ def reverse(self): ['c', 'b', 'a'] """ + # pylint: disable=protected-access # GrantJ 2019-03-22 Consider using an algorithm that swaps the values # at two keys. Like self._cache.swap(key1, key2, retry=True) The swap # method would exchange the values at two given keys. Then, using a From ef94856d2447fa9662bc62557f9b96ba6f131e15 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 17 Apr 2023 19:01:25 -0700 Subject: [PATCH 19/31] Bump version to 5.5.2 --- diskcache/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/diskcache/__init__.py b/diskcache/__init__.py index 95dafb3..134c88a 100644 --- a/diskcache/__init__.py +++ b/diskcache/__init__.py @@ -61,8 +61,8 @@ pass __title__ = 'diskcache' -__version__ = '5.5.1' -__build__ = 0x050501 +__version__ = '5.5.2' +__build__ = 0x050502 __author__ = 'Grant Jenks' __license__ = 'Apache 2.0' __copyright__ = 'Copyright 2016-2023 Grant Jenks' From 74e554c5d9340765f6fd6f7891da49a420e92b70 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 17 Apr 2023 19:05:01 -0700 Subject: [PATCH 20/31] Bump versions of checkout and setup-python --- .github/workflows/integration.yml | 8 ++++---- .github/workflows/release.yml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 07aceec..b596fc6 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -12,9 +12,9 @@ jobs: check: [bluecheck, doc8, docs, flake8, isortcheck, mypy, pylint, rstcheck] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: '3.11' - name: Install dependencies @@ -36,12 +36,12 @@ jobs: steps: - name: Set up Python ${{ matrix.python-version }} x64 - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} architecture: x64 - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Install tox run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 21b6e5b..efe73c6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Install libmemcached-dev run: | @@ -19,7 +19,7 @@ jobs: sudo apt-get install libmemcached-dev - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: '3.11' From 35dbeabd283b242e9afd33713a5cea5cd260f51d Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 17 Apr 2023 20:38:04 -0700 Subject: [PATCH 21/31] Add maxlen parameter to diskcache.Deque (#191) * Add maxlen parameter to diskcache.Deque --- diskcache/djangocache.py | 5 ++- diskcache/fanout.py | 5 ++- diskcache/persistent.py | 91 +++++++++++++++++++++++++++++++--------- docs/tutorial.rst | 3 ++ tests/test_deque.py | 27 +++++++++++- 5 files changed, 107 insertions(+), 24 deletions(-) diff --git a/diskcache/djangocache.py b/diskcache/djangocache.py index 8bf85ce..5dc8ce2 100644 --- a/diskcache/djangocache.py +++ b/diskcache/djangocache.py @@ -44,14 +44,15 @@ def cache(self, name): """ return self._cache.cache(name) - def deque(self, name): + def deque(self, name, maxlen=None): """Return Deque with given `name` in subdirectory. :param str name: subdirectory name for Deque + :param maxlen: max length (default None, no max) :return: Deque with given name """ - return self._cache.deque(name) + return self._cache.deque(name, maxlen=maxlen) def index(self, name): """Return Index with given `name` in subdirectory. diff --git a/diskcache/fanout.py b/diskcache/fanout.py index 5283490..50005fc 100644 --- a/diskcache/fanout.py +++ b/diskcache/fanout.py @@ -612,7 +612,7 @@ def cache(self, name, timeout=60, disk=None, **settings): _caches[name] = temp return temp - def deque(self, name): + def deque(self, name, maxlen=None): """Return Deque with given `name` in subdirectory. >>> cache = FanoutCache() @@ -626,6 +626,7 @@ def deque(self, name): 1 :param str name: subdirectory name for Deque + :param maxlen: max length (default None, no max) :return: Deque with given name """ @@ -641,7 +642,7 @@ def deque(self, name): disk=self._disk, eviction_policy='none', ) - deque = Deque.fromcache(cache) + deque = Deque.fromcache(cache, maxlen=maxlen) _deques[name] = deque return deque diff --git a/diskcache/persistent.py b/diskcache/persistent.py index c3f22b5..522bb74 100644 --- a/diskcache/persistent.py +++ b/diskcache/persistent.py @@ -75,7 +75,7 @@ class Deque(Sequence): """ - def __init__(self, iterable=(), directory=None): + def __init__(self, iterable=(), directory=None, maxlen=None): """Initialize deque instance. If directory is None then temporary directory created. The directory @@ -86,10 +86,11 @@ def __init__(self, iterable=(), directory=None): """ self._cache = Cache(directory, eviction_policy='none') - self.extend(iterable) + self._maxlen = float('inf') if maxlen is None else maxlen + self._extend(iterable) @classmethod - def fromcache(cls, cache, iterable=()): + def fromcache(cls, cache, iterable=(), maxlen=None): """Initialize deque using `cache`. >>> cache = Cache() @@ -111,7 +112,8 @@ def fromcache(cls, cache, iterable=()): # pylint: disable=no-member,protected-access self = cls.__new__(cls) self._cache = cache - self.extend(iterable) + self._maxlen = float('inf') if maxlen is None else maxlen + self._extend(iterable) return self @property @@ -124,6 +126,31 @@ def directory(self): """Directory path where deque is stored.""" return self._cache.directory + @property + def maxlen(self): + """Max length of the deque.""" + return self._maxlen + + @maxlen.setter + def maxlen(self, value): + """Set max length of the deque. + + Pops items from left while length greater than max. + + >>> deque = Deque() + >>> deque.extendleft('abcde') + >>> deque.maxlen = 3 + >>> list(deque) + ['c', 'd', 'e'] + + :param value: max length + + """ + self._maxlen = value + with self._cache.transact(retry=True): + while len(self._cache) > self._maxlen: + self._popleft() + def _index(self, index, func): len_self = len(self) @@ -244,7 +271,7 @@ def __iadd__(self, iterable): :return: deque with added items """ - self.extend(iterable) + self._extend(iterable) return self def __iter__(self): @@ -292,10 +319,11 @@ def __reversed__(self): pass def __getstate__(self): - return self.directory + return self.directory, self.maxlen def __setstate__(self, state): - self.__init__(directory=state) + directory, maxlen = state + self.__init__(directory=directory, maxlen=maxlen) def append(self, value): """Add `value` to back of deque. @@ -310,7 +338,12 @@ def append(self, value): :param value: value to add to back of deque """ - self._cache.push(value, retry=True) + with self._cache.transact(retry=True): + self._cache.push(value, retry=True) + if len(self._cache) > self._maxlen: + self._popleft() + + _append = append def appendleft(self, value): """Add `value` to front of deque. @@ -325,7 +358,12 @@ def appendleft(self, value): :param value: value to add to front of deque """ - self._cache.push(value, side='front', retry=True) + with self._cache.transact(retry=True): + self._cache.push(value, side='front', retry=True) + if len(self._cache) > self._maxlen: + self._pop() + + _appendleft = appendleft def clear(self): """Remove all elements from deque. @@ -340,6 +378,13 @@ def clear(self): """ self._cache.clear(retry=True) + _clear = clear + + def copy(self): + """Copy deque with same directory and max length.""" + TypeSelf = type(self) + return TypeSelf(directory=self.directory, maxlen=self.maxlen) + def count(self, value): """Return number of occurrences of `value` in deque. @@ -365,7 +410,9 @@ def extend(self, iterable): """ for value in iterable: - self.append(value) + self._append(value) + + _extend = extend def extendleft(self, iterable): """Extend front side of deque with value from `iterable`. @@ -379,7 +426,7 @@ def extendleft(self, iterable): """ for value in iterable: - self.appendleft(value) + self._appendleft(value) def peek(self): """Peek at value at back of deque. @@ -459,6 +506,8 @@ def pop(self): raise IndexError('pop from an empty deque') return value + _pop = pop + def popleft(self): """Remove and return value at front of deque. @@ -483,6 +532,8 @@ def popleft(self): raise IndexError('pop from an empty deque') return value + _popleft = popleft + def remove(self, value): """Remove first occurrence of `value` in deque. @@ -537,8 +588,8 @@ def reverse(self): # forward iterator and a reverse iterator, the reverse method could # avoid making copies of the values. temp = Deque(iterable=reversed(self)) - self.clear() - self.extend(temp) + self._clear() + self._extend(temp) directory = temp.directory temp._cache.close() del temp @@ -575,22 +626,22 @@ def rotate(self, steps=1): for _ in range(steps): try: - value = self.pop() + value = self._pop() except IndexError: return else: - self.appendleft(value) + self._appendleft(value) else: steps *= -1 steps %= len_self for _ in range(steps): try: - value = self.popleft() + value = self._popleft() except IndexError: return else: - self.append(value) + self._append(value) __hash__ = None # type: ignore @@ -669,7 +720,9 @@ def __init__(self, *args, **kwargs): args = args[1:] directory = None self._cache = Cache(directory, eviction_policy='none') - self.update(*args, **kwargs) + self._update(*args, **kwargs) + + _update = MutableMapping.update @classmethod def fromcache(cls, cache, *args, **kwargs): @@ -695,7 +748,7 @@ def fromcache(cls, cache, *args, **kwargs): # pylint: disable=no-member,protected-access self = cls.__new__(cls) self._cache = cache - self.update(*args, **kwargs) + self._update(*args, **kwargs) return self @property diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 1963635..69277d3 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -565,6 +565,9 @@ access and editing at both front and back sides. :class:`Deque 4 >>> other.popleft() 'foo' + >>> thing = Deque('abcde', maxlen=3) + >>> list(thing) + ['c', 'd', 'e'] :class:`Deque ` objects provide an efficient and safe means of cross-thread and cross-process communication. :class:`Deque ` diff --git a/tests/test_deque.py b/tests/test_deque.py index add7714..71c69e2 100644 --- a/tests/test_deque.py +++ b/tests/test_deque.py @@ -77,6 +77,20 @@ def test_getsetdel(deque): assert len(deque) == 0 +def test_append(deque): + deque.maxlen = 3 + for item in 'abcde': + deque.append(item) + assert deque == 'cde' + + +def test_appendleft(deque): + deque.maxlen = 3 + for item in 'abcde': + deque.appendleft(item) + assert deque == 'edc' + + def test_index_positive(deque): cache = mock.MagicMock() cache.__len__.return_value = 3 @@ -131,9 +145,12 @@ def test_state(deque): sequence = list('abcde') deque.extend(sequence) assert deque == sequence + deque.maxlen = 3 + assert list(deque) == sequence[-3:] state = pickle.dumps(deque) values = pickle.loads(state) - assert values == sequence + assert values == sequence[-3:] + assert values.maxlen == 3 def test_compare(deque): @@ -161,6 +178,14 @@ def test_repr(): assert repr(deque) == 'Deque(directory=%r)' % directory +def test_copy(deque): + sequence = list('abcde') + deque.extend(sequence) + temp = deque.copy() + assert deque == sequence + assert temp == sequence + + def test_count(deque): deque += 'abbcccddddeeeee' From 4beffe892a6c4352098a79614de40649d7e9f88e Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 17 Apr 2023 20:40:55 -0700 Subject: [PATCH 22/31] Bump version to 5.6.0 --- diskcache/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/diskcache/__init__.py b/diskcache/__init__.py index 134c88a..8647b9a 100644 --- a/diskcache/__init__.py +++ b/diskcache/__init__.py @@ -61,8 +61,8 @@ pass __title__ = 'diskcache' -__version__ = '5.5.2' -__build__ = 0x050502 +__version__ = '5.6.0' +__build__ = 0x050600 __author__ = 'Grant Jenks' __license__ = 'Apache 2.0' __copyright__ = 'Copyright 2016-2023 Grant Jenks' From cffbcec2b198e3a296ec294bd43da37fc559645b Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 17 Apr 2023 22:30:26 -0700 Subject: [PATCH 23/31] Fix docs re: JSONDisk --- docs/tutorial.rst | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 69277d3..2eb454d 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -821,30 +821,28 @@ example below uses compressed JSON, available for convenience as .. code-block:: python - import json, zlib - class JSONDisk(diskcache.Disk): def __init__(self, directory, compress_level=1, **kwargs): self.compress_level = compress_level - super(JSONDisk, self).__init__(directory, **kwargs) + super().__init__(directory, **kwargs) def put(self, key): json_bytes = json.dumps(key).encode('utf-8') data = zlib.compress(json_bytes, self.compress_level) - return super(JSONDisk, self).put(data) + return super().put(data) def get(self, key, raw): - data = super(JSONDisk, self).get(key, raw) + data = super().get(key, raw) return json.loads(zlib.decompress(data).decode('utf-8')) - def store(self, value, read): + def store(self, value, read, key=UNKNOWN): if not read: json_bytes = json.dumps(value).encode('utf-8') value = zlib.compress(json_bytes, self.compress_level) - return super(JSONDisk, self).store(value, read) + return super().store(value, read, key=key) def fetch(self, mode, filename, value, read): - data = super(JSONDisk, self).fetch(mode, filename, value, read) + data = super().fetch(mode, filename, value, read) if not read: data = json.loads(zlib.decompress(data).decode('utf-8')) return data From f81160f22af9e8af0e07e179808280188146a020 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 17 Apr 2023 22:46:44 -0700 Subject: [PATCH 24/31] Support pathlib.Path as directory argument --- diskcache/core.py | 1 + diskcache/fanout.py | 1 + tests/test_core.py | 8 ++++++++ tests/test_fanout.py | 8 ++++++++ 4 files changed, 18 insertions(+) diff --git a/diskcache/core.py b/diskcache/core.py index 9f3a597..af65454 100644 --- a/diskcache/core.py +++ b/diskcache/core.py @@ -433,6 +433,7 @@ def __init__(self, directory=None, timeout=60, disk=Disk, **settings): if directory is None: directory = tempfile.mkdtemp(prefix='diskcache-') + directory = str(directory) directory = op.expanduser(directory) directory = op.expandvars(directory) diff --git a/diskcache/fanout.py b/diskcache/fanout.py index 50005fc..9822ee4 100644 --- a/diskcache/fanout.py +++ b/diskcache/fanout.py @@ -30,6 +30,7 @@ def __init__( """ if directory is None: directory = tempfile.mkdtemp(prefix='diskcache-') + directory = str(directory) directory = op.expanduser(directory) directory = op.expandvars(directory) diff --git a/tests/test_core.py b/tests/test_core.py index 356d104..788afef 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -5,6 +5,7 @@ import io import os import os.path as op +import pathlib import pickle import shutil import sqlite3 @@ -37,6 +38,13 @@ def test_init(cache): cache.close() +def test_init_path(cache): + path = pathlib.Path(cache.directory) + other = dc.Cache(path) + other.close() + assert cache.directory == other.directory + + def test_init_disk(): with dc.Cache(disk_pickle_protocol=1, disk_min_file_size=2**20) as cache: key = (None, 0, 'abc') diff --git a/tests/test_fanout.py b/tests/test_fanout.py index deea03f..af221b6 100644 --- a/tests/test_fanout.py +++ b/tests/test_fanout.py @@ -5,6 +5,7 @@ import io import os import os.path as op +import pathlib import pickle import shutil import subprocess as sp @@ -44,6 +45,13 @@ def test_init(cache): cache.check() +def test_init_path(cache): + path = pathlib.Path(cache.directory) + other = dc.FanoutCache(path) + other.close() + assert cache.directory == other.directory + + def test_set_get_delete(cache): for value in range(100): cache.set(value, value) From 4d3068625a3edcd2f5a1f6f104ef621f1f7ea395 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Mon, 17 Apr 2023 22:47:22 -0700 Subject: [PATCH 25/31] Bump version to 5.6.1 --- diskcache/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/diskcache/__init__.py b/diskcache/__init__.py index 8647b9a..1931a0d 100644 --- a/diskcache/__init__.py +++ b/diskcache/__init__.py @@ -61,8 +61,8 @@ pass __title__ = 'diskcache' -__version__ = '5.6.0' -__build__ = 0x050600 +__version__ = '5.6.1' +__build__ = 0x050601 __author__ = 'Grant Jenks' __license__ = 'Apache 2.0' __copyright__ = 'Copyright 2016-2023 Grant Jenks' From 17a5f42facc312dae6e98b7b53345e2ed02be21d Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Wed, 30 Aug 2023 22:56:38 -0700 Subject: [PATCH 26/31] Bug fix: Fix peek when value is so large that a file is used (#288) Error caused by copy/paste from pull(). --- diskcache/core.py | 3 --- tests/test_deque.py | 10 ++++++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/diskcache/core.py b/diskcache/core.py index af65454..c7c8486 100644 --- a/diskcache/core.py +++ b/diskcache/core.py @@ -1703,9 +1703,6 @@ def peek( except IOError: # Key was deleted before we could retrieve result. continue - finally: - if name is not None: - self._disk.remove(name) break if expire_time and tag: diff --git a/tests/test_deque.py b/tests/test_deque.py index 71c69e2..f997a86 100644 --- a/tests/test_deque.py +++ b/tests/test_deque.py @@ -302,3 +302,13 @@ def test_rotate_indexerror_negative(deque): with mock.patch.object(deque, '_cache', cache): deque.rotate(-1) + + +def test_peek(deque): + value = b'x' * 100_000 + deque.append(value) + assert len(deque) == 1 + assert deque.peek() == value + assert len(deque) == 1 + assert deque.peek() == value + assert len(deque) == 1 From 63a5f6068b77fe9c02c8f310758fa1f05ae1ae04 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Wed, 30 Aug 2023 22:58:11 -0700 Subject: [PATCH 27/31] Bump version to 5.6.2 --- diskcache/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/diskcache/__init__.py b/diskcache/__init__.py index 1931a0d..719640f 100644 --- a/diskcache/__init__.py +++ b/diskcache/__init__.py @@ -61,8 +61,8 @@ pass __title__ = 'diskcache' -__version__ = '5.6.1' -__build__ = 0x050601 +__version__ = '5.6.2' +__build__ = 0x050602 __author__ = 'Grant Jenks' __license__ = 'Apache 2.0' __copyright__ = 'Copyright 2016-2023 Grant Jenks' From 23d10dce8f4be9c00df4786d508964b3b7d72b27 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Wed, 30 Aug 2023 23:09:54 -0700 Subject: [PATCH 28/31] Update release.yml to use pypa/gh-action-pypi-publish --- .github/workflows/release.yml | 29 ++++++++--------------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index efe73c6..33b3a8f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,35 +9,22 @@ jobs: upload: runs-on: ubuntu-latest + permissions: + id-token: write steps: - uses: actions/checkout@v3 - - name: Install libmemcached-dev - run: | - sudo apt-get update - sudo apt-get install libmemcached-dev - - name: Set up Python uses: actions/setup-python@v4 with: python-version: '3.11' - - name: Install dependencies - run: | - pip install --upgrade pip - pip install -r requirements-dev.txt - - - name: Create source dist - run: python setup.py sdist + - name: Install build + run: pip install build - - name: Create wheel dist - run: python setup.py bdist_wheel + - name: Create build + run: python -m build - - name: Upload with twine - env: - TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }} - TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} - run: | - ls -l dist/* - twine upload dist/* + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 From 323787f507a6456c56cce213156a78b17073fe00 Mon Sep 17 00:00:00 2001 From: Grant Jenks Date: Wed, 30 Aug 2023 23:10:27 -0700 Subject: [PATCH 29/31] Bump version to 5.6.3 --- diskcache/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/diskcache/__init__.py b/diskcache/__init__.py index 719640f..7757d66 100644 --- a/diskcache/__init__.py +++ b/diskcache/__init__.py @@ -61,8 +61,8 @@ pass __title__ = 'diskcache' -__version__ = '5.6.2' -__build__ = 0x050602 +__version__ = '5.6.3' +__build__ = 0x050603 __author__ = 'Grant Jenks' __license__ = 'Apache 2.0' __copyright__ = 'Copyright 2016-2023 Grant Jenks' From 9cd3816333fa34cb30d6cc2a7f227b6b1cdb793c Mon Sep 17 00:00:00 2001 From: ddorian Date: Tue, 27 Feb 2024 00:30:46 +0100 Subject: [PATCH 30/31] Change `Cache_expire_time` to a partial index because we don't need to query rows efficiently `where expire_time IS NULL` (#305) --- diskcache/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/diskcache/core.py b/diskcache/core.py index c7c8486..ad9ad4c 100644 --- a/diskcache/core.py +++ b/diskcache/core.py @@ -531,7 +531,7 @@ def __init__(self, directory=None, timeout=60, disk=Disk, **settings): sql( 'CREATE INDEX IF NOT EXISTS Cache_expire_time ON' - ' Cache (expire_time)' + ' Cache (expire_time) WHERE expire_time IS NOT NULL' ) query = EVICTION_POLICY[self.eviction_policy]['init'] From ebfa37cd99d7ef716ec452ad8af4b4276a8e2233 Mon Sep 17 00:00:00 2001 From: ddorian Date: Sun, 3 Mar 2024 02:19:29 +0100 Subject: [PATCH 31/31] Change `Cache_tag_rowid` to a partial index because we don't need to query rows efficiently `where tag IS NULL` (#307) * Change `Cache_tag_rowid` to a partial index because we don't need to query rows efficiently `where tag IS NULL` * Fix formatting --- diskcache/core.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/diskcache/core.py b/diskcache/core.py index ad9ad4c..7a3d23b 100644 --- a/diskcache/core.py +++ b/diskcache/core.py @@ -2028,7 +2028,10 @@ def create_tag_index(self): """ sql = self._sql - sql('CREATE INDEX IF NOT EXISTS Cache_tag_rowid ON Cache(tag, rowid)') + sql( + 'CREATE INDEX IF NOT EXISTS Cache_tag_rowid ON Cache(tag, rowid) ' + 'WHERE tag IS NOT NULL' + ) self.reset('tag_index', 1) def drop_tag_index(self):