Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 6 additions & 7 deletions mapillary_tools/history.py
Original file line number Diff line number Diff line change
Expand Up @@ -243,17 +243,16 @@ def set(self, key: str, value: str, expires_in: int = 3600 * 24 * 2) -> None:

@_retry_on_database_lock_error
def clear_expired(self) -> list[str]:
expired_keys: list[str] = []

s = time.perf_counter()

with self._lock:
with store.KeyValueStore(self._file, flag="c") as db:
for key, raw_payload in db.items():
data = self._decode(raw_payload)
if self._is_expired(data):
del db[key]
expired_keys.append(T.cast(str, key))
expired_keys = T.cast(
list[str],
db.delete_matching(
lambda raw_payload: self._is_expired(self._decode(raw_payload))
),
)

LOG.debug(
f"Cleared {len(expired_keys)} expired entries from the cache ({(time.perf_counter() - s) * 1000:.0f} ms)"
Expand Down
30 changes: 29 additions & 1 deletion mapillary_tools/store.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,9 @@
import os
import sqlite3
import sys
import typing as T
from collections.abc import MutableMapping
from contextlib import closing, suppress
from contextlib import closing, contextmanager, suppress
from pathlib import Path

BUILD_TABLE = """
Expand All @@ -30,6 +31,7 @@
STORE_KV = "REPLACE INTO Dict (key, value) VALUES (CAST(? AS BLOB), CAST(? AS BLOB))"
DELETE_KEY = "DELETE FROM Dict WHERE key = CAST(? AS BLOB)"
ITER_KEYS = "SELECT key FROM Dict"
ITER_ITEMS = "SELECT key, value FROM Dict"


def _normalize_uri(path):
Expand Down Expand Up @@ -120,6 +122,32 @@ def __iter__(self):
for row in cu:
yield row[0]

def delete_matching(self, predicate: T.Callable[[bytes], bool]) -> list[bytes]:
"""
Delete the items whose value matches the predicate and return their keys.

It is done in one transaction, so the database is synced to disk once,
not once per deleted item.
"""
with self._transaction():
with closing(self._cx.execute(ITER_ITEMS)) as cu:
keys = [key for key, value in cu if predicate(value)]
self._cx.executemany(DELETE_KEY, ((key,) for key in keys))
return keys

@contextmanager
def _transaction(self):
# Take the write lock up front, so other connections can not change
# the items between reading and writing them
self._cx.execute("BEGIN IMMEDIATE")
try:
yield
except BaseException:
if self._cx.in_transaction:
self._cx.execute("ROLLBACK")
raise
self._cx.execute("COMMIT")

def close(self):
self._cx.close()

Expand Down
2 changes: 0 additions & 2 deletions mapillary_tools/uploader.py
Original file line number Diff line number Diff line change
Expand Up @@ -590,8 +590,6 @@ def __init__(self, upload_options: UploadOptions, emitter: EventEmitter):
self.emitter = emitter
# Create a single shared SingleImageUploader instance that will be used across all uploads
cache = _maybe_create_persistent_cache_instance(self.upload_options)
if cache:
cache.clear_expired()
self.cached_image_uploader = CachedImageUploader(
self.upload_options, cache=cache
)
Expand Down
39 changes: 39 additions & 0 deletions tests/unit/test_store.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,45 @@ def test_basic_dict_operations(tmpdir):
assert "key2" in store


def test_delete_matching(tmpdir):
"""Test that delete_matching deletes the matching items in one transaction."""
db_path = tmpdir.join("test.db")

with KeyValueStore(str(db_path), flag="c") as store:
for idx in range(5):
store[f"key{idx}"] = "old" if idx < 3 else "new"

statements: list[str] = []
store._cx.set_trace_callback(statements.append)
deleted = store.delete_matching(lambda value: value == b"old")
store._cx.set_trace_callback(None)

assert sorted(deleted) == [b"key0", b"key1", b"key2"]
assert sorted(store.keys()) == [b"key3", b"key4"]

assert statements[0] == "BEGIN IMMEDIATE"
assert statements[-1] == "COMMIT"
assert sum(statement.startswith("DELETE") for statement in statements) == 3


def test_delete_matching_error(tmpdir):
"""Test that delete_matching deletes nothing if the predicate fails."""
db_path = tmpdir.join("test.db")

def predicate(value: bytes) -> bool:
raise ValueError(value)

with KeyValueStore(str(db_path), flag="c") as store:
store["key1"] = "value1"

with pytest.raises(ValueError):
store.delete_matching(predicate)

# The store is usable after the failed transaction
store["key2"] = "value2"
assert sorted(store.keys()) == [b"key1", b"key2"]


def test_keyerror_on_missing_key(tmpdir):
"""Test that KeyError is raised for missing keys."""
db_path = tmpdir.join("test.db")
Expand Down
Loading