Skip to content

Instantly share code, notes, and snippets.

@M4rtinK

M4rtinK/modRana tile storage rewrite WiP

Created September 15, 2015 22:18
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save M4rtinK/ee0bc31be41d8514c1d2 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save M4rtinK/ee0bc31be41d8514c1d2 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
modRana tile storage rewrite WiP
gedit
diff --git a/core/constants.py b/core/constants.py
index 6779fea..3c7f6f4 100644
--- a/core/constants.py
+++ b/core/constants.py
@@ -35,6 +35,7 @@ CONNECTIVITY_UNKNOWN = None
DEFAULT_TILE_STORAGE_TYPE = "files"
TILE_STORAGE_FILES = "files"
TILE_STORAGE_SQLITE = "sqlite"
+TILE_STORAGE_TYPES = [TILE_STORAGE_FILES, TILE_STORAGE_SQLITE]
# GTK GUI
PANGO_ON = '<span color="green">ON</span>'
diff --git a/core/tile_storage/base.py b/core/tile_storage/base.py
index bcaa53e..ac76afa 100644
--- a/core/tile_storage/base.py
+++ b/core/tile_storage/base.py
@@ -1,4 +1,4 @@
-"""Base classes for the Tile Storage module"""
+# Base classes for the Tile Storage module
import os
@@ -8,6 +8,16 @@ log = logging.getLogger("tile_storage.base")
class BaseTileStore(object):
"""An abstract class defining the tile store API"""
+ @staticmethod
+ def is_store(path):
+ """Report if given path is a tile store
+
+ :param str path: path to test
+ :returns: True if the path leads to a tile store, else False
+ :rtype: bool
+ """
+ pass
+
def __init__(self, store_path, prevent_media_indexing = False):
self._prevent_media_indexing = prevent_media_indexing
self._store_path = store_path
@@ -19,7 +29,10 @@ class BaseTileStore(object):
def store_tile_data(self, lzxy, tile_data):
pass
- def get_tile_data(self, lzxy):
+ def get_tile(self, lzxy):
+ pass
+
+ def has_tile(self, lzxy):
pass
def delete_tile(self, lzxy):
diff --git a/core/tile_storage/constants.py b/core/tile_storage/constants.py
index db8628c..6f59f99 100644
--- a/core/tile_storage/constants.py
+++ b/core/tile_storage/constants.py
@@ -1,3 +1 @@
-SQLITE_MAX_DB_FILE_SIZE = 3.7 # in Gibi Bytes
-SQLITE_QUEUE_SIZE = 50
-SQLITE_DEFAULT_STORAGE_FORMAT_VERSION = 1
\ No newline at end of file
+GIBI_BYTE = 2**30
diff --git a/core/tile_storage/files_store.py b/core/tile_storage/files_store.py
index 33cc866..c44859c 100644
--- a/core/tile_storage/files_store.py
+++ b/core/tile_storage/files_store.py
@@ -1,5 +1,233 @@
+from __future__ import with_statement
+import os
+import glob
+import shutil
+import re
+
+from .base import BaseTileStore
+from . import utils
+
import logging
log = logging.getLogger("tile_storage.files_store")
+PARTIAL_TILE_FILE_SUFFIX = ".part"
+
+def _get_toplevel_tile_folder_list(path):
+ """Return a list of toplevel tile folders
+ The toplevel tile folders correspond to the zoom level number.
+
+ :param str path: path where to check for top level tile folders
+ :returns: a list of top level tile folders
+ :rtype: list
+ """
+ tile_folders = [item for item in os.listdir(path) if re.match("[0-9]+", item)]
+ return [item for item in tile_folders if os.path.isdir(item)]
+
+
+class FileBasedTileStore(BaseTileStore):
+
+ @staticmethod
+ def is_store(path):
+ """We consider the path to be a files tile store if it is a folder
+ that contains at least one top level tile folder.
+ This basically means that at least one tile needs to be stored in it.
+
+ :param str path: path to test
+ :returns: True if the path leads to an existing folder, else False
+ :rtype: bool
+ """
+ is_files_store = False
+ if os.path.isdir(path):
+ is_files_store = bool(_get_toplevel_tile_folder_list(path))
+
+ def __init__(self, store_path, prevent_media_indexing = False):
+ BaseTileStore.__init__(self, store_path, prevent_media_indexing=prevent_media_indexing)
+
+ def store_tile_data(self, lzxy, tile_data):
+ """Store the given tile to a file"""
+ # get the folder path
+ file_path = self._get_tile_file_path(lzxy)
+ partial_file_path = file_path + PARTIAL_TILE_FILE_SUFFIX
+ (folder_path, tail) = os.path.split(file_path)
+ if not os.path.exists(folder_path): # does it exist ?
+ try:
+ os.makedirs(folder_path) # create the folder
+ except Exception:
+ import sys
+ e = sys.exc_info()[1]
+ # errno 17 - folder already exists
+ # this is most probably cased by another thread creating the folder between
+ # the check & our os.makedirs() call
+ # -> we can safely ignore it (as the the only thing we are now interested in,
+ # is having a folder to store the tile in)
+ if e.errno != 17:
+ log.exception("can't create folder %s for %s", folder_path, file_path)
+ try:
+ with open(partial_file_path, 'wb') as f:
+ f.write(tile_data)
+ os.rename(partial_file_path, file_path)
+ # TODO: fsync the file (optionally ?)?
+ except:
+ log.exception("saving tile to file %s failed", file_path)
+ try:
+ if os.path.exists(partial_file_path):
+ os.remove(partial_file_path)
+ if os.path.exists(file_path):
+ os.remove(file_path)
+ except:
+ log.exception("failed storage operation cleanup failed for %s", file_path)
+
+ def get_tile(self, lzxy, fuzzy_matching=False):
+ """Get tile data and timestamp corresponding to the given coordinate tuple from
+ this file based tile store. The timestamp correspond to the time the tile has
+ been last modified.
+
+ :param tuple lzxy: layer, z, x, y coordinate tuple describing a single tile
+ (layer is actually not used and can be None)
+ :param bool fuzzy_matching: if fuzzy tile matching should be used
+ Fuzzy tile matching looks for any image files on the given coordinates
+ in case an image file with extension given by the layer.type property is
+ not found in the store.
+ :returns: (tile data, timestamp) or None if tile is not found in the database
+ :rtype: a (bytes, int) tuple or None
+ """
+ if fuzzy_matching:
+ file_path = self._fuzzy_find_tile(lzxy)
+ else:
+ file_path = os.path.isfile(self._get_tile_file_path(lzxy))
+
+ if file_path:
+ try:
+ tile_mtime = os.path.getmtime(file_path)
+ with open(file_path, "rb") as f:
+ return f.read(), tile_mtime
+ except:
+ log.exception("tile file reading failed for: %s", file_path)
+ return None
+ else:
+ return None
+
+ def has_tile(self, lzxy, fuzzy_matching=False):
+ """Report if the tile specified by the lzxy tuple is present
+ in this file based tile store
+
+ :param tuple lzxy: layer, z, x, y coordinate tuple describing a single tile
+ :param bool fuzzy_matching: if fuzzy tile matching should be used
+ Fuzzy tile matching looks for any image files on the given coordinates
+ in case an image file with extension given by the layer.type property is
+ not found in the store.
+ :returns: True if tile is present in the store, else False
+ :rtype: bool
+ """
+ if fuzzy_matching:
+ return bool(self._fuzzy_find_tile(lzxy))
+ else:
+ return os.path.isfile(self._get_tile_file_path(lzxy))
+
+ def _delete_empty_folders(self, z, x):
+ # x-level folder
+ x_path = os.path.join(self.store_path, z, x)
+ if not os.listdir(x_path):
+ try:
+ try:
+ os.rmdir(x_path)
+ except OSError:
+ # most probably caused by something being created in the folder
+ # since the check
+ pass
+ # z-level folder
+ z_path = os.path.join(self.store_path, z)
+ if os.listdir(z_path):
+ try:
+ os.rmdir(z_path)
+ except OSError:
+ # most probably caused by something being created in the folder
+ # since the check
+ pass
+ except:
+ log.exception("empty folder cleanup failed for: %s", x_path)
+
+ def delete_tile(self, lzxy):
+ # TODO: delete empty folders ?
+ tile_path = self._get_tile_file_path(lzxy)
+ try:
+ if os.path.isfile(tile_path):
+ os.remove(tile_path)
+ # remove any empty folders that might have been
+ # left after the deleted tile file
+ self._delete_empty_folders(lzxy[1], lzxy[2])
+ else:
+ log.error("can't delete file - path is not a file: %s", tile_path)
+ except:
+ log.exception("removing of tile at %s failed", tile_path)
+
+ def clear(self):
+ """Delete the tile folders and files corresponding to this store from permanent storage
+ This basically means we delete all folders that have just number in name
+ (1, 2, 23, 1337 but not aa1, b23, .git, etc.) as they are the the toplevel (z coordinates)
+ tile storage folders.
+ We don't just remove the toplevel folder as for example in modRana a single folder
+ is used to store both tiles and sqlite tile storage database and we would
+ also remove any databases if we just removed the toplevel folder.
+ """
+ try:
+ for folder in _get_toplevel_tile_folder_list(self.store_path):
+ folder_path = os.path.join(self.store_path, folder)
+ shutil.rmtree(folder_path)
+ except:
+ log.exception("clearing of files tile store at path %s failed", self.store_path)
+
+ def _get_tile_file_path(self, lzxy):
+ """Return full filesystem path to the tile file corresponding to the coordinates
+ given by the lzxy tuple.
+
+ :param tuple lzxy: tile coordinates
+ :returns: full filesystem path to the tile
+ :rtype: str
+ """
+ return os.path.join(
+ self.store_path,
+ str(lzxy[1]),
+ str(lzxy[2]),
+ "%d.%s" % (lzxy[3], lzxy[0].type)
+ )
+
+ def _fuzzy_find_tile(self, lzxy):
+ """Try to find a tile image file for the given coordinates
+
+ :returns: path to a suitable tile or None if no can be found
+ :rtype: str or None
+ """
+ # first check if the primary tile path exists
+ tile_path = self._get_tile_file_path(lzxy)
+ # check if the primary file path exists and also if it actually
+ # is an image file
+ if os.path.exists(tile_path):
+ try:
+ with open(tile_path, "rb") as f:
+ if utils.is_an_image(f.read(32)):
+ return tile_path
+ else:
+ log.warning("%s is not an image", tile_path)
+ except Exception:
+ log.exception("checking if primary tile file is an image failed for %s", lzxy)
+ # look also for other supported image formats
+ alternative_tile_path = None
+ # TODO: might be good to investigate the performance impact,
+ # just to be sure :P
+ # replace the extension with a * so that the path can be used
+ # as wildcard in glob
+ wildcard_tile_path = "%s.*" % os.path.splitext(tile_path)[0]
+ # iterate over potential paths returned by the glob iterator
+ for path in glob.iglob(wildcard_tile_path):
+ # go over the paths and check if they point to an image files
+ with open(path, "rb") as f:
+ if utils.is_an_image(f.read(32)):
+ # once an image file is found, break & returns its path
+ alternative_tile_path = path
+ break
+ else:
+ log.warning("%s is not an image", tile_path)
+ return alternative_tile_path
\ No newline at end of file
diff --git a/core/tile_storage/sqlite_store.py b/core/tile_storage/sqlite_store.py
index 69d7b9a..0221c12 100644
--- a/core/tile_storage/sqlite_store.py
+++ b/core/tile_storage/sqlite_store.py
@@ -1,3 +1,42 @@
+# A tile store that is using SQLite as a storage backend
+#
+# This has a number of benefits:
+# * all the tiles are stored in just a couple files
+# - usually just 2 (1 one lookup and 1 storage database)
+# - additional storage database is added for every 3.7 GB of tiles by default
+# * improves performance on filesystems that fail to handle many small file efficiently (FAT32)
+# * makes occupied space checking instant
+# (it can take a lot of time to compute size of tile stored as files in folders)
+# * hides the tiles from stupid multimedia indexers
+#
+# How does it work ?
+# The tiles are stored in a sqlite database as blobs. There are two types of database files,
+# lookup.sqlite and store.sqlite.
+# The lookup file has the name of the storage database where the requested tile data is stored.
+# The store file has the actual tile data.
+# Multiple stores should be numbered in ascending order, starting from 0:
+#
+# store.sqlite.0
+# store.sqlite.1
+# store.sqlite.2
+# etc.
+#
+# The storage database looks schema like this:
+#
+# table tiles (z integer, x integer, y integer, store_filename string, extension varchar(10), unix_epoch_timestamp integer, primary key (z, x, y, extension))
+#
+# The storage databases schema look like this:
+#
+# table tiles (z integer, x integer, y integer, tile blob, extension varchar(10), unix_epoch_timestamp integer, primary key (z, x, y, extension))
+#
+# The only difference in the structure is that the lookup databases only stores the name of the store
+# for given coordinates and the store database stores the actual blob.
+# Both also have a table called version which has an integer column called v.
+# There is a single 1 inserted, which indicates the current version of the table.
+#
+# When looking for a tile in the database, the lookup database is checked first and if the coordinates
+# are found the corresponding storage database is queried for the actual data.
+
from __future__ import with_statement
import os
@@ -11,6 +50,7 @@ log = logging.getLogger("tile_storage.sqlite_store")
from .base import BaseTileStore
from .constants import GIBI_BYTE
+from . import utils
# the storage database files can be only this big to avoid
# maximum file size limitations on FAT32 and possibly elsewhere
@@ -18,6 +58,7 @@ MAX_STORAGE_DB_FILE_SIZE = 3.7 # in Gibi Bytes
SQLITE_QUEUE_SIZE = 50
SQLITE_TILE_STORAGE_FORMAT_VERSION = 1
LOOKUP_DB_NAME = "lookup.sqlite"
+STORE_DB_NAME_PREFIX = "store.sqlite."
def connect_to_db(path_to_database):
"""Setting check_same_thread to False fixes a Sqlite exception
@@ -43,13 +84,33 @@ def connect_to_db(path_to_database):
class SqliteTileStore(BaseTileStore):
+ @staticmethod
+ def is_store(path):
+ """We consider the path to be a sqlite tile store if it:
+ - if a folder
+ - contains the lookup database and at least one storage database
+
+ :param str path: path to test
+ :returns: True if the path leads to sqlite tile store, else False
+ :rtype: bool
+ """
+ is_store = False
+ if os.path.isdir(path):
+ lookup_db_path = os.path.join(path, LOOKUP_DB_NAME)
+ store_db_glob = os.path.join(path, STORE_DB_NAME_PREFIX + "*")
+ if os.path.isfile(lookup_db_path) and glob.glob(store_db_glob):
+ # we have found the lookup database and at least one store,
+ # so this is most probably a sqlite tile database
+ is_store = True
+ return is_store
+
def __init__(self, store_path, prevent_media_indexing = False):
BaseTileStore.__init__(self, store_path, prevent_media_indexing=prevent_media_indexing)
# SQLite tends to blow up with the infamous "sqlite3.OperationalError: database is locked"
# if the database is accessed from multiple threads an/or processes at the same time.
# Interestingly this can happen if only reading is going on - if the RO transaction
- # is taking too long, attemps of writing will start getting the "database is locked"
+ # is taking too long, attempts of writing will start getting the "database is locked"
# error. And of course a single write transaction is enough to basically block the
# database indefinitely.
#
@@ -59,58 +120,43 @@ class SqliteTileStore(BaseTileStore):
# which should ot be that difficult as for example modRana often needs to fetch a full
# screen of tiles and we can quite easily cache write requests and submit them in batch.
self._db_lock = RLock()
+ # to avoid possible race conditions we needs to mutually exclude operations concerning
+ # storage database free space checking and the related adding of new stores
+ self._storage_db_management_lock = RLock()
# there is always only one lookup database per store that stores only tile coordinates
# and name of the storage database holding the tile data
# (hitting the 4 GB file size limit while storing only tile coordinates
# should hopefully never happen)
self._lookup_db_path = os.path.join(self.store_path, LOOKUP_DB_NAME)
- self._lookup_db_connection = connect_to_db(self._lookup_db_path)
+ self._lookup_db_connection = self._get_lookup_db_connection()
# there is always one or more storage databases that hold the actual tile data
- # - once a storage database hits the
- self._storage_connections = {}
-
- def get_lookup_db_path(self, db_folder_path):
- return os.path.join(db_folder_path, ) # get the path to the lookup db
-
- def _initialize_storage_db(self, folder_prefix, access_type):
- """There are two access types, "store" and "get"
- we basically have two connection tables, store and get ones
- the store connections are used only by the worker to store all the tiles it gets to its queue
- the get connections are used by the main thread
- when other threads query the database, they create a new connection and disconnect it when done
-
- we still get "database locked" from time to time,
- * it may bee needed to use a mutex to control db access to fix this
- * or disconnect all database connections ?
+ # - once a storage database hits the max file size limit (actually se to 3.7 GB just in case)
+ # a new storage database file is added
+ self._storage_databases = self._get_storage_db_connections()
+
+ # these two variables point to a storage database that currently has enough free space
+ # for storing of new tiles, once the storage database hits the max file size limit,
+ # we try to find one that has enough free space or add a new one if all existing
+ # storage databases are full
+ # - we just connect the storage with the lowest number at startup and let the usable
+ # storage database finding logic do its job in case it is too full
+ sorted_store_names = list(sorted(self._storage_databases.keys()))
+ store_name, store_connection = self._storage_databases[sorted_store_names[0]]
+ self._new_tiles_store_name = store_name
+ self._new_tiles_store_connection = store_connection
+
+ def _get_lookup_db_connection(self):
+ """Initialize the lookup database
+ If the database already exist just connect to it, otherwise create it and
+ in both cases return the database connection.
"""
- db_folder_path = self.store_path
- if access_type in self._layers: # do we have an entry for this access type ?
- if db_folder_path in self._layers[access_type]:
- return db_folder_path # return the lookup connection
- else:
- self._initialize_lookup_db(folder_prefix, access_type) # initialize the lookup db
- return db_folder_path
- else:
- self._layers[access_type] = {} # create a new access type entry
- self._initialize_lookup_db(folder_prefix, access_type) # initialize the lookup db
- return db_folder_path
-
- def _initialize_lookup_db(self, folder_prefix, access_type):
- """Initialize the lookup database"""
- db_folder_path = self.store_path
-
- if not os.path.exists(db_folder_path): # does the folder exist ?
- os.makedirs(db_folder_path) # create the folder
-
- lookup_db_path = self.get_lookup_db_path(db_folder_path) # get the path
-
- if db_folder_path not in self._layers.keys():
- log.info("sqlite tiles: initializing db for layer: %s" % folder_prefix)
- if os.path.exists(lookup_db_path): #does the lookup db exist ?
- connection = connect_to_db(lookup_db_path) # connect to the lookup db
- else: #create new lookup db
- connection = connect_to_db(lookup_db_path)
+ log.debug("initializing lookup db: %s" % self._lookup_db_path)
+ if os.path.exists(self._lookup_db_path): #does the lookup db exist ?
+ connection = connect_to_db(self._lookup_db_path) # connect to the lookup db
+ else: # create new lookup database
+ with self._db_lock:
+ connection = connect_to_db(self._lookup_db_path)
cursor = connection.cursor()
log.info("sqlite tiles: creating lookup table")
cursor.execute(
@@ -118,310 +164,277 @@ class SqliteTileStore(BaseTileStore):
cursor.execute("create table version (v integer)")
cursor.execute("insert into version values (?)", (SQLITE_TILE_STORAGE_FORMAT_VERSION,))
connection.commit()
- self._layers[access_type][db_folder_path] = {'lookup': connection, 'stores': {}}
-
- def _store_tile_to_sqlite(self, tile, lzxy):
- layer, z, x, y = lzxy
- folder_prefix = layer.folderName
- extension = layer.type
- access_type = "store"
- db_folder_path = self._initialize_storage_db(folder_prefix, access_type)
- if db_folder_path is not None:
- lookup_conn = self._layers[access_type][db_folder_path]['lookup'] # connect to the lookup db
- stores = self._layers[access_type][db_folder_path]['stores'] # get a list of cached store connections
- lookup_cursor = lookup_conn.cursor()
- with self._lookup_connection_lock:
- # just to make sure the access is sequential
- # (due to sqlite in python 2.5 probably not liking concurrent access,
- # resulting in te database becoming unavailable)
- result = lookup_cursor.execute(
- "select store_filename, unix_epoch_timestamp from tiles where z=? and x=? and y=?",
- (z, x, y))
- if not result.fetchone():
- # store the tile as its not already in the database
-
- # get a store path
- size = len(tile)
- path_to_store = self._get_an_available_store_path(folder_prefix, size)
-
- # connect to the store
- store_conn = self._connect_to_store(stores, path_to_store, db_folder_path, access_type)
-
- # store the tile
- integer_timestamp = int(time.time())
- try:
- ## 1. write in the lookup db (its easier to remove pointers to nonexistent stuff than to remove orphaned store items)
- store_filename = os.path.basename(path_to_store)
- lookup_query = "insert into tiles (z, x, y, store_filename, extension, unix_epoch_timestamp) values (?, ?, ?, ?, ?, ?)"
- lookup_cursor = lookup_conn.cursor()
- lookup_cursor.execute(lookup_query, [z, x, y, store_filename, extension, integer_timestamp])
-
- ## 2. write in the store
- store_query = "insert into tiles (z, x, y, tile, extension, unix_epoch_timestamp) values (?, ?, ?, ?, ?, ?)"
- store_cursor = store_conn.cursor()
- store_cursor.execute(store_query,
- [z, x, y, sqlite3.Binary(tile), extension, integer_timestamp])
- self.commit_connections([store_conn, lookup_conn])
- except Exception:
- log.exception("tile already present")
- #tile is already present, skip insert
-
- def commit_connections(self, connections):
- """Store connections and commit them once in a while"""
- for c in connections:
- self.dirty.add(c)
-
-
- def commit_all(self, tilesInCommit=0):
- """Commit all uncommitted"""
- with self._lookup_connection_lock:
- # just to make sure the access is sequential
- # (due to sqlite in python 2.5 probably not liking concurrent access,
- # resulting in te database becoming unavailable)
- while self.dirty:
- conn = self.dirty.pop()
- conn.commit()
- log.debug("sqlite commit OK (%d tiles)", tilesInCommit)
-
-
- def _connect_to_store(self, stores, path_to_store, db_folder_path, access_type):
- """Connect to a store
- * use an existing connection or create a new one"""
- if path_to_store in stores.keys():
- return stores[path_to_store] # there is already a connection to the store, return it
- else: # create a new connection
- with self._lookup_connection_lock:
- # just to make sure the access is sequential
- # (due to sqlite in python 2.5 probably not liking concurrent access,
- # resulting in te database becoming unavailable)
- storeConn = connect_to_db(path_to_store) #TODO: add some error handling
- self._layers[access_type][db_folder_path]['stores'][path_to_store] = storeConn # cache the connection
- return storeConn
-
- def _get_an_available_store_path(self, folder_prefix, size):
- """Return a path to a store that can be used to store a tile specified by size"""
- layer_db_folder_path = self.store_path
- store_list = self._list_stores(layer_db_folder_path)
- if store_list: # there are already some stores
- available_stores = []
- for store_path in store_list: # iterate over the available stores
- clean_path = store_path.rstrip('-journal') # don't add sqlite journal files
- if self._will_it_fit_in(clean_path, size):# add all stores that can be used to store the current object
- available_stores.append(clean_path)
-
- if available_stores: # did we find some stores ?
- available_store_path = available_stores.pop() # use one of the stores
- else:
- available_store_path = self._add_store(layer_db_folder_path) # all stores full, add a new one
-
- else: # there are no stores, add one
- available_store_path = self._add_store(layer_db_folder_path)
-
- return available_store_path
-
- def _list_stores(self, folder):
- """Search a folder for available store database files"""
- return glob.glob(os.path.join(folder, "store.sqlite.*"))
-
-
- def _will_it_fit_in(self, storage_database_path, size_in_bytes):
+ return connection
+
+ def _get_storage_db_connections(self):
+ """Connect to all existing storage databases and return a dictionary of the resulting connections
+ - if no storage databases exist, create the first (store.sqlite.0) storage database
+ """
+ connections = {}
+ existing_stores = self._list_stores()
+ if existing_stores:
+ for store_path in existing_stores:
+ connections[store_path] = connect_to_db(store_path)
+ else: # no stores yet, create the first one
+ store_name, store_connection = self._add_store()
+ connections = {store_name : store_connection}
+ return connections
+
+ def _add_store(self):
+ """Add a new store
+ - find an ascending numbered name and create the db file with the corresponding tables
+ """
+ new_highest_number = 0
+ storeList = self._list_stores()
+ if storeList:
+ number_candidate_list = map(lambda x: x.split('store.sqlite.')[1], storeList)
+ integer_list = []
+ for number_candidate in number_candidate_list:
+ try:
+ integer_list.append(int(number_candidate))
+ except ValueError:
+ # there was probably something like store.sqlite.abc,
+ # eq. something that fails to parse to an integer
+ pass
+ if integer_list:
+ highest_number = sorted(number_candidate_list)[-1]
+ new_highest_number = highest_number + 1
+
+ store_name = "store.sqlite.%d" % new_highest_number
+ return store_name, self._create_new_store(os.path.join(self.store_path, store_name))
+
+ def _create_new_store(self, path):
+ """Create a new store database at the given file path
+
+ :param str path: path to the file path where the database should be created
+ """
+ log.debug("creating a new storage database in %s" % path)
+ connection = connect_to_db(path)
+ cursor = connection.cursor()
+ cursor.execute(
+ "create table tiles (z integer, x integer, y integer, tile blob, extension varchar(10), unix_epoch_timestamp integer, primary key (z, x, y, extension))")
+ cursor.execute("create table version (v integer)")
+ cursor.execute("insert into version values (?)", (SQLITE_TILE_STORAGE_FORMAT_VERSION,))
+ connection.commit()
+ return connection
+
+ def _get_name_connection_to_available_store(self, data_size):
+ """Return a path to a store that can be used to store a tile specified by its size"""
+ with self._storage_db_management_lock:
+ # first check if the last-known-good storage database has enough space to satisfy the request
+ if self._will_it_fit_in(self._new_tiles_store_name, data_size):
+ return self._new_tiles_store_name, self._new_tiles_store_connection
+ else: # try if the request will fit to some other store we are already connected to
+ for store_name, store_connection in self._storage_databases.items():
+ if self._will_it_fit_in(store_name, data_size):
+ # this store can handle the given storage request, so set it as able to
+ # handle further tile storage requests
+ self._new_tiles_store_name = store_name
+ self._new_tiles_store_connection = store_connection
+ # and return the connection to caller
+ return store_name, store_connection
+
+ # if we got there it means we have not found space for the request in any existing
+ # storage database file, so we need to create a new one
+ new_store_name, new_store_connection = self._add_store()
+ # again cache the connection to the store
+ self._new_tiles_store_name = new_store_name
+ self._new_tiles_store_connection = new_store_connection
+ return new_store_name, new_store_connection
+
+ def _list_stores(self):
+ """Return a list of available storage database files
+
+ :returns: list of storage database paths
+ :rtype: list of strings
+ """
+ return glob.glob(os.path.join(self.store_path, "%s*" % STORE_DB_NAME_PREFIX))
+
+ def _will_it_fit_in(self, storage_database_name, size_in_bytes):
"""Report if the given amount of data in bytes will still fit into the currently used
storage database
True = probably fits in the database
False = would not fit in
NOTE: there is some database overhead, so this is not 100% reliable
always set the limit with a slight margin
- """
+ :param str storage_database_name: path to a storage database to check
+ :param int size_in_bytes: the size to check
+ :returns: True if size will fit, False if not
+ :rtype: bool
+ """
maximum_size_in_bytes = MAX_STORAGE_DB_FILE_SIZE * GIBI_BYTE
+ storage_database_path = os.path.join(self.store_path, storage_database_name)
store_size_in_bytes = os.path.getsize(storage_database_path)
if (store_size_in_bytes + size_in_bytes) <= maximum_size_in_bytes:
return True # the database will (probably) still smaller than the limit
else:
return False # the database will be larger
- def _add_store(self, folder):
- """Add a new store(find an ascending numbered name and create the db file with the corresponding tables)"""
- storeList = self._list_stores(folder)
- if storeList:
- numericList = map(lambda x: int(x.split('store.sqlite.')[1]), storeList)
- highestNumber = sorted(numericList)[-1]
- newHighestNumber = highestNumber + 1
- else:
- newHighestNumber = 0
-
- storeName = "store.sqlite.%d" % newHighestNumber
- newStorePath = self._get_store_path(folder, storeName)
- return self._create_new_store(newStorePath)
-
- def _create_new_store(self, path):
- """Create a new store table and file"""
- log.info("sqlite tiles: creating a new storage database in %s" % path)
- os.path.exists(path)
- connection = connect_to_db(path)
- cursor = connection.cursor()
- cursor.execute(
- "create table tiles (z integer, x integer, y integer, tile blob, extension varchar(10), unix_epoch_timestamp integer, primary key (z, x, y, extension))")
- cursor.execute("create table version (v integer)")
- cursor.execute("insert into version values (?)", (SQLITE_TILE_STORAGE_FORMAT_VERSION,))
- connection.commit()
- return path
-
- def _get_store_path(self, folder, store_name):
- """Get a standardized store path from folder path and filename"""
- return os.path.join(folder, store_name)
-
-
- def getTileFromDb(self, lookupConn, dbFolderPath, lzxy):
- """get a tile from the database"""
- accessType = "get"
- layer, z, x, y = lzxy
- #look in the lookup db
- #with self.lookupConnectionLock:
- if 1:
- # just to make sure the access is sequential
- # (due to sqlite in python 2.5 probably not liking concurrent access,
- # resulting in te database becoming unavailable)
- lookupCursor = lookupConn.cursor()
- lookupResult = lookupCursor.execute(
+ def store_tile_data(self, lzxy, tile_data):
+ with self._db_lock:
+ layer, z, x, y = lzxy
+ extension = layer.type
+ lookup_connection = self._lookup_db_connection
+ lookup_cursor = lookup_connection.cursor()
+ data_size = len(tile_data)
+ integer_timestamp = int(time.time())
+ tile_exists = lookup_cursor.execute(
+ "select store_filename from tiles where z=? and x=? and y=?",
+ (z, x, y)).fetchone()
+ if tile_exists: # tile is already in the database, update it
+ # check if the new tile will fit to the storage database where the tile currently is
+ # (we count as we would add the tile to the database, not replace it du to
+ # database file size uncertainties caused by metadata updates, etc.)
+ store_name = tile_exists[0]
+ if self._will_it_fit_in(store_name, data_size):
+ # update the tile data and its timestamp in place
+ store_connection = self._storage_databases[store_name]
+ store_cursor = store_connection.cursor()
+ # update the storage database
+ su_query = "insert or replace into tiles (z, x, y, tile, extension, unix_epoch_timestamp) values (?, ?, ?, ?, ?, ?)"
+ # use "insert or replace" in case that the storage database is missing the tile for some reason
+ # - this should never happen as long as the database is properly managed, but better be safe than sorry
+ store_cursor.execute(su_query, [z, x, y, sqlite3.Binary(tile_data), extension, integer_timestamp])
+ store_connection.commit()
+ # update the extension and timestamp in the lookup database
+ lu_query = "update tiles set extension=?, unix_epoch_timestamp=? where z=? and x=? and y=?"
+ lookup_cursor.execute(lu_query, [extension, integer_timestamp, z, x, y])
+ lookup_connection.commit()
+ else:
+ # remove the tile from the current storage database file
+ old_store_connection = self._storage_databases[store_name]
+ old_store_cursor = old_store_connection.cursor()
+ old_store_cursor.execute("delete from tiles where z=? and x=? and y=?", (z, x, y))
+ old_store_connection.commit()
+ # find a suitable storage database file
+ new_store_name, new_store_connection = self._get_name_connection_to_available_store(data_size)
+ # store the tile to it
+ store_query = "insert or replace into tiles (z, x, y, tile, extension, unix_epoch_timestamp) values (?, ?, ?, ?, ?, ?)"
+ # we use "insert or replace" in case there already is an unexpected leftover tile in the store for the coordinates
+ # - this should never happen as long as the database is properly managed, but better be safe than sorry
+ store_cursor = new_store_connection.cursor()
+ store_cursor.execute(store_query, [z, x, y, sqlite3.Binary(tile_data), extension, integer_timestamp])
+ new_store_connection.commit()
+ # update the store path, extension and timestamp in the lookup database
+ lu_query = "update tiles set store_filename=?, extension=?, unix_epoch_timestamp=? where where z=? and x=? and y=?"
+ lookup_cursor.execute(lu_query, [new_store_name, extension, integer_timestamp, z, x, y])
+ lookup_connection.commit()
+
+ else: # tile is not yet in the database, so just store it
+ # get a store that can store this tile
+ store_name, store_connection = self._get_name_connection_to_available_store(data_size)
+ # write in the lookup db
+ lookup_query = "insert into tiles (z, x, y, store_filename, extension, unix_epoch_timestamp) values (?, ?, ?, ?, ?, ?)"
+ lookup_cursor = lookup_connection.cursor()
+ lookup_cursor.execute(lookup_query, [z, x, y, store_name, extension, integer_timestamp])
+ lookup_connection.commit()
+ # write in the store
+ store_query = "insert into tiles (z, x, y, tile, extension, unix_epoch_timestamp) values (?, ?, ?, ?, ?, ?)"
+ store_cursor = store_connection.cursor()
+ store_cursor.execute(store_query, [z, x, y, sqlite3.Binary(tile_data), extension, integer_timestamp])
+ store_connection.commit()
+
+ def get_tile(self, lzxy):
+ """Get tile data and timestamp corresponding to the given coordinate tuple from the database.
+ The timestamp correspond to the time the tile has been last modified.
+
+ :param tuple lzxy: layer, z, x, y coordinate tuple describing a single tile
+ (layer is actually not used and can be None)
+ :returns: (tile data, timestamp) or None if tile is not found in the database
+ :rtype: a (bytes, int) tuple or None
+ """
+ _layer, z, x, y = lzxy
+ with self._db_lock:
+ lookup_connection = self._lookup_db_connection
+ lookup_cursor = lookup_connection.cursor()
+ lookup_result = lookup_cursor.execute(
"select store_filename, unix_epoch_timestamp from tiles where z=? and x=? and y=?",
(z, x, y)).fetchone()
- if lookupResult: # the tile was found in the lookup db
- # now search in the store
- storeFilename = lookupResult[0]
- pathToStore = self.get_store_path(dbFolderPath, storeFilename)
- stores = self._layers[accessType][dbFolderPath]['stores']
- connectionToStore = self.connect_to_store(stores, pathToStore, dbFolderPath, accessType)
- storeCursor = connectionToStore.cursor()
+ if lookup_result: # the tile was found in the lookup db
+ # now search for in the specified store
+ store_name = lookup_result[0]
+ store_connection = self._storage_databases.get(store_name)
+ if store_connection is None:
+ log.warning("store %s/%s is mentioned in lookup db for %s/%s/%s but does not exist",
+ self.store_path, store_name, z, x, y)
+ return None
+ store_cursor = store_connection.cursor()
# as the x,y & z are used as the primary key, all rows need to have a unique
# x, y & z combination and thus there can be only one result for a select
# over x, y & z
- result = storeCursor.execute(
+ result = store_cursor.execute(
"select tile, unix_epoch_timestamp from tiles where z=? and x=? and y=?",
(z, x, y))
- if utils.isTheStringAnImage(str(result)):
- self.log.warning("%s,%s,%s in %s is probably not an image", x, y, z, dbFolderPath)
+ if not utils.is_an_image(str(result)):
+ log.warning("%s,%s,%s in %s/%s is probably not an image", x, y, z, self.store_path, store_name)
return result
- else: # the tile was not found in the lookup table
+ else: # the tile was not found in the lookup database
return None
- def tileExists2(self, lzxy, fromThread=False):
- """Test if a tile exists
- if fromThread=False, a new connection is created and disconnected again
- NEW CLEANED UP VERSION
+ def delete_tile(self, lzxy):
+ """Try to delete tile corresponding to the lzxy coordinate tuple from the database
- TODO: automatically check the function is being
- called from a non-main thread
+ :param tuple lzxy: layer, z, x, y coordinate tuple describing a single tile
+ (layer is actually not used and can be None)
"""
- layer, z, x, y = lzxy
- storageType = self.get('tileStorageType', self.modrana.dmod.defaultTileStorageType)
- if storageType == 'sqlite': # we are storing to the database
- dbFolderPath = self.get_layer_db_folder_path(layer.folderName)
- if dbFolderPath is not None: # is the database accessible ?
- with self._lookup_connection_lock:
- # just to make sure the access is sequential
- # (due to sqlite in python 2.5 probably not liking concurrent access,
- # resulting in te database becoming unavailable)
- if fromThread: # is this called from a thread ?
- # due to sqlite quirks, connections can't be shared between threads
- lookupDbPath = self.get_lookup_db_path(dbFolderPath)
- lookupConn = sqliteConnectionWrapper(lookupDbPath)
- else:
- # TODO: check if the database is actually initialized for the given layer
- accessType = "get"
- lookupConn = self._layers[accessType][dbFolderPath]['lookup'] # connect to the lookup db
- query = "select store_filename, unix_epoch_timestamp from tiles where z=? and x=? and y=?"
- lookupResult = lookupConn.execute(query, (z, x, y)).fetchone()
- if fromThread: # tidy up, just to be sure
- lookupConn.close()
- if lookupResult:
- return True # the tile is in the db
- else:
- return False # the tile is not in the db
- else:
- return None # we cant decide if a tile is ind the db or not
- else: # we are storing to the filesystem
- return os.path.exists(self._get_tile_file_path(lzxy))
-
- def tileExists(self, filePath, lzxy, fromThread=False):
- """test if a tile exists
- if fromThread=False, a new connection is created and disconnected again"""
- storageType = self.get('tileStorageType', self.modrana.dmod.defaultTileStorageType)
- layer, z, x, y = lzxy
- if storageType == 'sqlite': # we are storing to the database
- dbFolderPath = self.get_layer_db_folder_path(layer.folderName)
- if dbFolderPath is not None: # is the database accessible ?
- with self._lookup_connection_lock:
- # just to make sure the access is sequential
- # (due to sqlite in python 2.5 probably not liking concurrent access,
- # resulting in te database becoming unavailable)"""
- if fromThread: # is this called from a thread ?
- # due to sqlite quirks, connections can't be shared between threads
- lookupDbPath = self.get_lookup_db_path(dbFolderPath)
- lookupConn = sqliteConnectionWrapper(lookupDbPath)
- else:
- lookupConn = self._layers[dbFolderPath]['lookup'] # connect to the lookup db
- query = "select store_filename, unix_epoch_timestamp from tiles where z=? and x=? and y=?"
- lookupResult = lookupConn.execute(query, (z, x, y)).fetchone()
- if fromThread: # tidy up, just to be sure
- lookupConn.close()
- if lookupResult:
- return True # the tile is in the db
- else:
- return False # the tile is not in the db
- else:
- return None # we cant decide if a tile is ind the db or not
- else: # we are storing to the filesystem
- return os.path.exists(filePath)
-
- def _start_tile_loading_thread(self):
- """Start the sqlite loading thread"""
- t = threads.ModRanaThread(target=self._tile_loader,
- name=constants.THREAD_TILE_STORAGE_LOADER)
- # we need that the worker tidies up,
- # (commits all "dirty" connections)
- # so it should be not daemonic
- # -> but we also cant afford that modRana wont
- # terminate completely
- # (all ModRanaThreads are daemonic by default)
- threads.threadMgr.add(t)
-
- def _tile_loader(self):
- """This is run by a thread that stores sqlite tiles to a db"""
- tilesInCommit = 0
- while True:
- try:
- item = self._sqlite_tile_queue.get(block=True)
- except Exception:
- # this usually occurs during interpreter shutdown
- # -> we simulate a shutdown order and try to exit cleanly
- self.log.exception("storage thread - probable shutdown")
- item = 'shutdown'
-
- if item == 'shutdown': # we put this to the queue to announce shutdown
- self.log.info("shutdown imminent, committing all uncommitted tiles")
- self.commit_all(tilesInCommit)
- self.log.info("all tiles committed, breaking, goodbye :)")
- break
- # the thread should not die due to an exception
- # or the queue fills up without anybody removing and processing the tiles
- # -> this would mean that all threads that need to store tiles
- # would wait forever for the queue to empty
- #TODO: defensive programming - check if thread is alive when storing ?
- try:
- (tile, lzxy) = item # unpack the tuple
- self._store_tile_to_sqlite(tile, lzxy) # store the tile
- tilesInCommit+=1
- self._sqlite_tile_queue.task_done()
- except Exception:
- self.log.exception("sqlite storage worker: exception during tile storage")
-
- dt = time.time() - self._last_commit # check when the last commit was
- if dt > self._commit_interval:
- try:
- self.commit_all(tilesInCommit) # commit all "dirty" connections
- tilesInCommit = 0
- self._last_commit = time.time() # update the last commit timestamp
- except Exception:
- self.log.exception("sqlite storage worker: exception during mass db commit")
-
+ with self._db_lock:
+ _layer, z, x, y = lzxy
+ lookup_connection = self._lookup_db_connection
+ lookup_cursor = lookup_connection.cursor()
+ store_name = lookup_cursor.execute(
+ "select store_filename from tiles where z=? and x=? and y=?", (z, x, y)
+ ).fetchone()[0]
+ if store_name:
+ store_connection = self._storage_databases[store_name]
+ store_cursor = store_connection.cursor()
+ store_cursor.execute("delete from tiles where z=? and x=? and y=?", (z, x, y))
+ store_connection.commit()
+ lookup_cursor.execute("delete from tiles where z=? and x=? and y=?", (z, x, y))
+ lookup_connection.commit()
+
+ def has_tile(self, lzxy):
+ """Report if a tile specified by the lzxy tuple is stored in the database
+
+ NOTE: We only check in the lookup database, not in the storage databases and we also
+ don't verify if the tile extension is the same one as specified by the layer object
+ of the lzxy tuple.
+
+ :param tuple lzxy: layer, z, x, y coordinate tuple describing a single tile
+ (layer is actually not used and can be None)
+ :returns: True if tile is stored in the database, else False
+ :rtype: bool
+ """
+ _layer, z, x, y = lzxy
+ lookup_connection = self._lookup_db_connection
+ lookup_cursor = lookup_connection.cursor()
+ query = "select store_filename, unix_epoch_timestamp from tiles where z=? and x=? and y=?"
+ lookupResult = lookup_cursor.execute(query, (z, x, y)).fetchone()
+ if lookupResult:
+ return True # the tile is in the database
+ else:
+ return False # the tile is not in the database
+
+ def close(self):
+ """Close all database connections"""
+ with self._db_lock:
+ self._lookup_db_connection.close()
+ for connection in self._storage_databases.values():
+ connection.close()
+
+ def clear(self):
+ """Delete all database files belonging to this SQLite store"""
+ with self._db_lock:
+ # make sure the connections are closed before we remove
+ # the data bases under them
+ self.close()
+ with self._storage_db_management_lock:
+ # delete the lookup database
+ os.remove(self._lookup_db_path)
+ self._lookup_db_path = None
+ self._lookup_db_connection = None
+ # delete the storage databases
+ for db_name in self._storage_databases.keys():
+ os.remove(os.path.join(self.store_path, db_name))
+ self._storage_databases = {}
+ # TODO: the database should be able to handle writes after clear
diff --git a/core/tile_storage/utils.py b/core/tile_storage/utils.py
index c40fc75..41fc230 100644
--- a/core/tile_storage/utils.py
+++ b/core/tile_storage/utils.py
@@ -6,7 +6,7 @@ from .tile_types import ID_TO_CLASS_MAP
PYTHON3 = sys.version_info[0] > 2
-def is_data_an_image(tile_data):
+def is_an_image(tile_data):
"""test if the string contains an image
by reading its magic number"""
@@ -35,4 +35,4 @@ def is_data_an_image(tile_data):
return False
def get_tile_data_type(tile_data):
- return ID_TO_CLASS_MAP.get(is_data_an_image(tile_data), None)
+ return ID_TO_CLASS_MAP.get(is_an_image(tile_data), None)
diff --git a/modules/mod_options/mod_options.py b/modules/mod_options/mod_options.py
index 27e1dd1..bb18b6c 100644
--- a/modules/mod_options/mod_options.py
+++ b/modules/mod_options/mod_options.py
@@ -558,8 +558,8 @@ class Options(RanaModule):
# ** tile storage
group = addGroup("Tile storage", "tile_storage", catMap, "generic")
addOpt("Tile storage", "tileStorageType",
- [('files', "files (default, more space used)"),
- ('sqlite', "sqlite (new, less space used)")],
+ [(constants.TILE_STORAGE_FILES, "files (default, more space used)"),
+ (constants.TILE_STORAGE_SQLITE, "sqlite (new, less space used)")],
group,
self.modrana.dmod.defaultTileStorageType)
addBoolOpt("Store downloaded tiles", "storeDownloadedTiles", group, True)
diff --git a/modules/mod_storeTiles.py b/modules/mod_storeTiles.py
index 2fc0987..a1c7804 100644
--- a/modules/mod_storeTiles.py
+++ b/modules/mod_storeTiles.py
@@ -34,17 +34,26 @@ except Exception:
import os
import time
import glob
+import collections
+
+try: # Python 2.7+
+ from collections import OrderedDict as OrderedDict
+except ImportError:
+ from core.backports.odict import odict as OrderedDict # Python <2.7
+
+
try:
import Queue
except ImportError:
import queue as Queue
-import threading
+from threading import RLock
from core import threads
from core import constants
from core import utils
-
+from core.tile_storage.files_store import FileBasedTileStore
+from core.tile_storage.sqlite_store import SqliteTileStore
def getModule(*args, **kwargs):
return StoreTiles(*args, **kwargs)
@@ -56,220 +65,181 @@ class StoreTiles(RanaModule):
def __init__(self, *args, **kwargs):
RanaModule.__init__(self, *args, **kwargs)
- self._layers = {}
- self._thread_layers = {}
- self._current_sqlite_storage_version = 1
- self._max_db_file_size_in_gibi_bytes = 3.7 # maximum database file size (to avoid the FAT32 file size limitation) in GibiBytes
- self._max_tiles_in_queue = 50
- self._sqlite_tile_queue = Queue.Queue(self._max_tiles_in_queue)
- # if there are more tiles in the buffer than maxTilesInBuffer,
- # the whole buffer will be processed at once (flushed) to avoid
- # a potential memory leak
-
- # how often we commit to the database (only happens when there is something in the queue)
- self._commit_interval = int(self.get("sqliteTileDatabaseCommitInterval",
- constants.DEFAULT_SQLITE_TILE_DATABASE_COMMIT_INTERVAL))
- self.log.info("sqlite tile database commit interval: %d s", self._commit_interval)
- self._last_commit = time.time()
- self.dirty = set() # a set of connections, that have uncommitted data
- # locks
- # TODO: this lock might not be needed for python2.6+,
- # as their sqlite version should be thread safe
+ self._tile_storage_management_lock = RLock()
+ # this variable holds the currently selected primary store type
+ # - this store type will be used to store all downloaded tiles
+ # - this store type will be tried first when looking for tiles
+ # in local storage, before checking other store types if the
+ # tile is not found
+ # - we have a watch on the store type key, so this variable will
+ # be automatically updated if store type changes at runtime
+ self._primary_tile_storage_type = constants.DEFAULT_TILE_STORAGE_TYPE
+ self._stores = collections.defaultdict(self._get_existing_stores_for_layer)
- self._lookup_connection_lock = threading.RLock()
+ self._prevent_media_indexing = self.dmod.getDeviceIDString() == "android"
# the tile loading debug log function is no-op by default, but can be
# redirected to the normal debug log by setting the "tileLoadingDebug"
# key to True
- self._loadingLog = self._noOp
- self.modrana.watch('tileLoadingDebug', self._tileLoadingDebugChangedCB, runNow=True)
+ self._llog = self._noOp
+ self.modrana.watch('tileLoadingDebug', self._tile_loading_debug_changed_cb, runNow=True)
+ self.modrana.watch('tileStorageType', self._primary_tile_storage_type_changed_cb, runNow=True)
# device modules are loaded and initialized and configs are parsed before "normal"
# modRana modules are initialized, so we can cache the map folder path in init
- self._mapFolderPath = self.modrana.paths.getMapFolderPath()
- # check if the folder exists and create it if it does not exist yet and do any other
- # operations that might need to be done on the folder to make it usable for storing
- # map data, such as preventing indexing of the tile images, etc.
- self._checkMapFolder()
- def firstTime(self):
- self._start_tile_loading_thread()
+ def _get_existing_stores_for_layer(self, layer):
+ """Check for any existing stores for the given layer in persistent storage
+ and return a dictionary with the found stores under file storage type keys.
+ """
+ self._llog("looking for existing stores for layer %s" % layer)
+ layer_folder_path = os.path.join(self.modrana.paths.getMapFolderPath(), layer.folderName)
+ store_tuples = []
+ # check if the path contains a file based tile store
+ if FileBasedTileStore.is_store(layer_folder_path):
+ self._llog("file based store has been found for layer %s" % layer)
+ # we need to prevent tiles from being indexed to the gallery
+ # - this basically dumps a .nomedia file to the root of the
+ # file based tile store folder
+ store_tuple = (constants.TILE_STORAGE_FILES, FileBasedTileStore(
+ layer_folder_path, prevent_media_indexing=self._prevent_media_indexing
+ ))
+ store_tuples.append(store_tuple)
+ # check if the path contains a sqlite tile store
+ if SqliteTileStore.is_store(layer_folder_path):
+ self._llog("sqlite tile store has been found for layer %s" % layer)
+ store_tuple = (constants.TILE_STORAGE_SQLITE, SqliteTileStore(layer_folder_path))
+ store_tuples.append(store_tuple)
+
+ self._llog("%d existing stores have been found for layer %s" % (len(store_tuples), layer))
+ # sort the tuples so that the primary tile storage type (if any) is first
+ store_tuples.sort(key=self._sort_store_tuples)
+ return OrderedDict(store_tuples)
+
+ def _get_stores_for_reading(self, layer):
+ """Get an iterable of stores for the given layer
+ - store corresponding to primary storage type is always first (if any)
+ - there might not by any stores (eq. no tiles) for the given layer
+ """
+ with self._tile_storage_management_lock:
+ return self._stores[layer].values()
- def _tileLoadingDebugChangedCB(self, key, oldValue, newValue):
+ def _get_store_for_writing(self, layer):
+ """Get a store for writing tiles corresponding to the given layer
+ and current primary tile storage type.
+ """
+ with self._tile_storage_management_lock:
+ store = self._stores[layer].get(self._primary_tile_storage_type)
+ if store is None:
+ self._llog("store type %s not found for layer %s" % (self._primary_tile_storage_type, layer))
+ layer_folder_path = os.path.join(self.modrana.paths.getMapFolderPath(), layer.folderName)
+ if self._primary_tile_storage_type is constants.TILE_STORAGE_FILES:
+ store_tuple = (constants.TILE_STORAGE_FILES, FileBasedTileStore(
+ layer_folder_path, prevent_media_indexing=self._prevent_media_indexing
+ ))
+ self._llog("adding file based store for layer %s" % layer)
+ else: # sqlite tile store
+ store_tuple = (constants.TILE_STORAGE_SQLITE, SqliteTileStore(layer_folder_path))
+ self._llog("adding file based store for layer %s" % layer)
+ # add the store to the stores dict while keeping the primary-storage-type first ordering
+ self._add_store_for_layer(layer, store_tuple)
+ self._llog("added store type %s for layer %s" % (self._primary_tile_storage_type, layer))
+ return store
+
+ def _sort_store_tuples(self, key):
+ # stores corresponding to the current primary stile storage type should go first
+ if key[0] == self._primary_tile_storage_type:
+ return 0
+ else:
+ return 1
+
+ def _sort_layer_odict(self, layer):
+ """Sort the ordered dict for the given layer according to
+ current primary tile storage type.
+ """
+ with self._tile_storage_management_lock:
+ store_tuples = list(self._stores[layer].items())
+ store_tuples.sort(key=self._sort_store_tuples)
+ self._stores[layer] = OrderedDict(store_tuples)
+
+ def _add_store_for_layer(self, layer, store_tuple):
+ """Add a store for the given layer while keeping the ordering
+ according to the current primary tile storage type."""
+ with self._tile_storage_management_lock:
+ store_tuples = list(self._stores[layer].items())
+ store_tuples.append(store_tuple)
+ store_tuples.sort(key=self._sort_store_tuples)
+ self._stores[layer] = OrderedDict(store_tuples)
+
+ def _primary_tile_storage_type_changed_cb(self, key, oldValue, newValue):
+ # primary tile storage type has been changed, update internal variables accordingly
+ with self._tile_storage_management_lock:
+ self._llog("primary tile storage type changed to: %s" % newValue)
+ if newValue not in constants.TILE_STORAGE_TYPES:
+ log.error("invalid tile storage type was requested and will be ignored: %s", newValue)
+ return
+ self._primary_tile_storage_type = newValue
+ # reorder the ordered dicts storing already initialized tile stores
+ # so that the primary tile storage method is first
+ self._llog("resorting ordered dicts for the new primary storage type")
+ for layer in self._stores.keys():
+ self._sort_layer_odict(layer)
+
+ def _tile_loading_debug_changed_cb(self, key, oldValue, newValue):
if newValue:
- self._loadingLog = self.log.debug
+ self._llog = self.log.debug
else:
- self._loadingLog = self._noOp
+ self._llog = self._noOp
def _noOp(self, *args):
pass
-
- def getTileData(self, lzxy):
- """
- return data for the given tile
- """
- storageType = self.get('tileStorageType', self.modrana.dmod.defaultTileStorageType)
- if storageType == 'sqlite':
- return self._getTileDataFromSqlite(lzxy)
- else: # the only other storage method is currently the classical files storage
- return self._getTileDataFromFiles(lzxy)
-
- def _getTileDataFromSqlite(self, lzxy):
- self._loadingLog("looking for tile in sqlite %s", lzxy)
- accessType = "get"
- layer = lzxy[0] # only the layer part of the tuple
- if layer.timeout:
- # stored timeout is in hours, convert to seconds
- timeout = float(layer.timeout)*60*60
- else:
- timeout = 0
- dbFolderPath = self.initialize_storage_db(lzxy[0].folderName, accessType)
- if dbFolderPath is not None:
- lookupConn = self._layers[accessType][dbFolderPath]['lookup'] # connect to the lookup db
- result = self.getTileFromDb(lookupConn, dbFolderPath, lzxy)
- if result: # is the result valid ?
- resultData = result.fetchone() # get the result content
- else:
- resultData = None # the result is invalid
- if resultData:
+ def get_tile_data(self, lzxy):
+ layer = lzxy[0]
+ with self._tile_storage_management_lock:
+ stores = self._get_stores_for_reading(layer)
+ for store in stores:
+ tile_tuple = store.get_tile(lzxy)
+ if tile_tuple is not None:
+ tile_data, timestamp = tile_tuple
if layer.timeout:
- self.log.debug("timeout set for layer %s: %fs (expired at %d), tile timestamp: %d" % (layer.label,timeout,time.time()-timeout,resultData[1]) )
- if resultData[1] < (time.time()-timeout):
- self.log.debug("tile is older than configured timeout, not loading tile")
+ # stored timeout is in hours, convert to seconds
+ layer_timeout = float(layer.timeout)*60*60
+ dt = time.time() - layer_timeout
+ self._llog("timeout set for layer %s: %fs (expired at %d), "
+ "tile timestamp: %d" % (layer.label,
+ layer_timeout,
+ dt,
+ timestamp))
+ if timestamp < dt:
+ self.log.debug("not loading timeod-out tile: %s" % lzxy)
return None # pretend the tile is not stored
- # return tile data
- return resultData[0]
- else:
- return None # the tile is not stored
-
- def _getTileDataFromFiles(self, lzxy):
- self._loadingLog("looking for tile in files %s", lzxy)
- layer = lzxy[0] # only the layer part of the tuple
- if layer.timeout:
- # stored timeout is in hours, convert to seconds
- timeout = float(layer.timeout)*60*60
- else:
- timeout = 0
- tilePath = self._fuzzy_tile_file_exists(lzxy)
- if tilePath:
- self._loadingLog("tile file found %s (path: %s)", lzxy, tilePath)
- if layer.timeout:
- tile_mtime = os.path.getmtime(tilePath)
- if tile_mtime < (time.time()-timeout):
- self.log.debug("file is older than configured timeout of %fs, not loading tile" % timeout)
- return None
- # load the file to pixbuf and return it
- try:
- with open(tilePath, "rb") as f:
- self._loadingLog("returning data for %s", lzxy)
- return f.read()
- except Exception:
- self.log.exception("loading tile from file failed for %s", lzxy)
- return None
- else:
- self._loadingLog("path does not exist for %s", lzxy)
- return None # this tile is not locally stored
-
-
-
- def automaticStoreTile(self, tile, lzxy):
- """store a tile to a file or db, depending on the current setting"""
+ else: # still fresh enough
+ return tile_data
+ else: # the tile is always fresh
+ return tile_data
+ # nothing found in any store (or no stores)
+ return None
- # check if persistent tile storage is enabled ?
- if self.get('storeDownloadedTiles', True):
- storageType = self.get('tileStorageType', self.modrana.dmod.defaultTileStorageType)
- if storageType == 'sqlite': # we are storing to the database
- # put the tile to the storage queue, so that then worker can store it
- self._sqlite_tile_queue.put((tile, lzxy), block=True, timeout=20)
- else: # we are storing to the filesystem
- self._store_tile_to_file(tile, lzxy)
-
- def _store_tile_to_file(self, tile, lzxy):
- """Store the given tile to a file"""
- # get the folder path
- filePath = self._get_tile_file_path(lzxy)
- (folderPath, tail) = os.path.split(filePath)
- if not os.path.exists(folderPath): # does it exist ?
- try:
- os.makedirs(folderPath) # create the folder
- except Exception:
- import sys
- e = sys.exc_info()[1]
- # errno 17 - folder already exists
- # this is most probably cased by another thread creating the folder between
- # the check & our os.makedirs() call
- # -> we can safely ignore it (as the the only thing we are now interested in,
- # is having a folder to store the tile in)
- if e.errno != 17:
- self.log.exception("can't create folder %s for %s", folderPath, filePath)
- try:
- with open(filePath, 'wb') as f:
- f.write(tile)
- except:
- self.log.exception("saving tile to file %s failed", filePath)
-
- def _get_tile_file_path(self, lzxy):
- """Return full filesystem path to the tile file corresponding to the coordinates
- given by the lzxy tuple.
-
- :param tuple lzxy: tile coordinates
- :returns: full filesystem path to the tile
- :rtype: str
- """
- return os.path.join(
- self._mapFolderPath,
- lzxy[0].folderName,
- str(lzxy[1]),
- str(lzxy[2]),
- "%d.%s" % (lzxy[3], lzxy[0].type)
- )
-
- def _fuzzy_tile_file_exists(self, lzxy):
- """Try to find a tile image file for the given coordinates."""
- # first check if the primary tile path exists
- tilePath = self._get_tile_file_path(lzxy)
- # check if the primary file path exists and also if it actually
- # is an image file
- if os.path.exists(tilePath):
- try:
- with open(tilePath, "rb") as f:
- if utils.isTheStringAnImage(f.read(32)):
- return tilePath
- else:
- self.log.warning("%s is not an image", tilePath)
- except Exception:
- self.log("error when checking if primary tile file is an image")
-
- # look also for other supported image formats
- alternativeTilePath = None
- # TODO: might be good to investigate the performance impact,
- # just to be sure :P
-
- # replace the extension with a * so that the path can be used
- # as wildcard in glob
- wildcardTilePath = "%s.*" % os.path.splitext(tilePath)[0]
- # iterate over potential paths returned by the glob iterator
- for path in glob.iglob(wildcardTilePath):
- # go over the paths and check if they point to an image files
- with open(path, "rb") as f:
- if utils.isTheStringAnImage(f.read(32)):
- # once an image file is found, break & returns its path
- alternativeTilePath = path
- break
- else:
- self.log.warning("%s is not an image", tilePath)
- return alternativeTilePath
+ def store_tile_data(self, lzxy, tile_data):
+ store = self._get_store_for_writing(lzxy[0])
+ store.store_tile_data(lzxy, tile_data)
def shutdown(self):
- # try to commit possibly uncommitted tiles
- self._sqlite_tile_queue.put('shutdown', True)
-
+ # close all stores
+ self.log.debug("closing tile stores")
+ store_count = 0
+ with self._tile_storage_management_lock:
+ for store_odicts in self._stores.values():
+ for store in store_odicts.values():
+ store.close()
+ store_count+=1
+ self.log.debug("closed %d tile stores")
## TESTING CODE
# accessType = "get"
# folderPrefix = 'OpenStreetMap I'
-# dbFolderPath = self._initialize_storage_db(folderPrefix, accessType)
+# dbFolderPath = self._get_storage_db_connections(folderPrefix, accessType)
## lookupConn = self.layers[accessType][dbFolderPath]['lookup']
# start = time.clock()
#
@@ -280,7 +250,7 @@ class StoreTiles(RanaModule):
# (z, x, y, extension) = tile
#
# lookupDbPath = self.get_lookup_db_path(dbFolderPath)
-# lookupConn = sqlite_connection_wrapper(lookupDbPath)
+# lookupConn = connect_to_db(lookupDbPath)
# lookupCursor = lookupConn.cursor()
# lookupResult = lookupCursor.execute("select store_filename, unix_epoch_timestamp from tiles where z=? and x=? and y=? and extension=?", (z, x, y, extension)).fetchone()
# lookupConn.close()
@@ -288,7 +258,7 @@ class StoreTiles(RanaModule):
#
# start = time.clock()
# lookupDbPath = self.get_lookup_db_path(dbFolderPath)
-# lookupConn = sqlite_connection_wrapper(lookupDbPath)
+# lookupConn = connect_to_db(lookupDbPath)
# lookupCursor = lookupConn.cursor()
# for tile in temp:
# (z, x, y, extension) = tile
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment