Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# Copyright (C) 2018 Bloomberg LP 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# <http://www.apache.org/licenses/LICENSE-2.0> 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14 

15 

16""" 

17LRUMemoryCache 

18================== 

19 

20A storage provider that stores data in memory. When the size limit 

21is reached, items are deleted from the cache with the least recently 

22used item being deleted first. 

23""" 

24 

25import collections 

26import io 

27import logging 

28import threading 

29 

30from .storage_abc import StorageABC 

31 

32 

33class _NullBytesIO(io.BufferedIOBase): 

34 """A file-like object that discards all data written to it.""" 

35 

36 def writable(self): 

37 return True 

38 

39 def write(self, b): 

40 return len(b) 

41 

42 

43class LRUMemoryCache(StorageABC): 

44 

45 def __init__(self, limit): 

46 self.__logger = logging.getLogger(__name__) 

47 

48 self._limit = limit 

49 self._storage = collections.OrderedDict() 

50 self._bytes_stored = 0 

51 self._lock = threading.Lock() 

52 self.logger = logging.getLogger(__name__) 

53 

54 def has_blob(self, digest): 

55 self.__logger.debug(f"Checking for blob: [{digest}]") 

56 with self._lock: 

57 key = (digest.hash, digest.size_bytes) 

58 result = key in self._storage 

59 if result: 

60 self._storage.move_to_end(key) 

61 return result 

62 

63 def get_blob(self, digest): 

64 self.__logger.debug(f"Getting blob: [{digest}]") 

65 with self._lock: 

66 key = (digest.hash, digest.size_bytes) 

67 if key in self._storage: 

68 self._storage.move_to_end(key) 

69 return io.BytesIO(self._storage[key]) 

70 return None 

71 

72 def delete_blob(self, digest): 

73 self.__logger.debug(f"Deleting blob: [{digest}]") 

74 key = (digest.hash, digest.size_bytes) 

75 self._storage.pop(key, None) 

76 

77 def begin_write(self, digest): 

78 if digest.size_bytes > self._limit: 

79 # Don't try to cache objects bigger than our memory limit. 

80 return _NullBytesIO() 

81 return io.BytesIO() 

82 

83 def commit_write(self, digest, write_session): 

84 self.__logger.debug(f"Writing blob: [{digest}]") 

85 if isinstance(write_session, _NullBytesIO): 

86 # We can't cache this object, so return without doing anything. 

87 return 

88 with self._lock: 

89 self._bytes_stored += digest.size_bytes 

90 

91 if self._bytes_stored > self._limit: 

92 # Delete stuff until we're back under the limit. 

93 while self._bytes_stored > self._limit: 

94 deleted_key = self._storage.popitem(last=False)[0] 

95 self._bytes_stored -= deleted_key[1] 

96 

97 key = (digest.hash, digest.size_bytes) 

98 self._storage[key] = write_session.getvalue()