diff --git a/settings.py b/settings.py index 92a3c47..83f6dcc 100644 --- a/settings.py +++ b/settings.py @@ -62,14 +62,14 @@ HAYSTACK_CUSTOM_HIGHLIGHTER = 'project.search.Highlighter' HAYSTACK_DEFAULT_OPERATOR = 'OR' HAYSTACK_CONNECTIONS = { 'default': { - 'ENGINE': 'haystack.backends.whoosh_backend.WhooshEngine', + 'ENGINE': 'project.filestore.S3WhooshEngine', 'PATH': os.path.join('/whoosh', 'default'), }, } for lang_code, lang in LANGUAGES: HAYSTACK_CONNECTIONS[lang_code] = {} HAYSTACK_CONNECTIONS[lang_code].update({ - 'ENGINE': 'haystack.backends.whoosh_backend.WhooshEngine', + 'ENGINE': 'project.filestore.S3WhooshEngine', 'PATH': os.path.join('/whoosh', lang_code), }) diff --git a/src/project/filestore.py b/src/project/filestore.py new file mode 100644 index 0000000..99fe31a --- /dev/null +++ b/src/project/filestore.py @@ -0,0 +1,133 @@ +from threading import Lock + +from aldryn_django.storage import S3MediaStorage +from haystack.backends.whoosh_backend import WhooshEngine, WhooshSearchBackend +from whoosh import index +from whoosh.filedb.filestore import Storage, ReadOnlyError + +import os +from whoosh.filedb.structfile import StructFile +from whoosh.qparser import QueryParser +from whoosh.util import random_name + +media_storage = S3MediaStorage() + + +class S3FileStorage(Storage): + supports_mmap = False + + def __init__(self, path, readonly=False, debug=False): + self.folder = path + self.readonly = readonly + self._debug = debug + self.locks = {} + + def __repr__(self): + return "%s(%r)" % (self.__class__.__name__, self.folder) + + def create(self): + return self + + def destroy(self): + del self.locks + self.clean() + + def create_file(self, name, excl=False, mode="wb", **kwargs): + if self.readonly: + raise ReadOnlyError + + path = self._fpath(name) + f = StructFile(media_storage.open(path, mode=mode), name=name, **kwargs) + return f + + def open_file(self, name, **kwargs): + path = self._fpath(name) + f = StructFile(media_storage.open(path, 'rb'), name=name, **kwargs) + return f + + def _fpath(self, fname): + return os.path.abspath(os.path.join(self.folder, fname)) + + def clean(self, ignore=False): + if self.readonly: + raise ReadOnlyError + + files = self.list() + for name in files: + self.delete_file(name) + + def list(self): + dirs, files = media_storage.listdir(self.folder) + return files + + def file_exists(self, name): + return media_storage.exists(self._fpath(name)) + + def file_modified(self, name): + return media_storage.get_modified_time(self._fpath(name)) + + def file_length(self, name): + return media_storage.size(self._fpath(name)) + + def delete_file(self, name): + if self.readonly: + raise ReadOnlyError + + media_storage.delete(self._fpath(name)) + + def rename_file(self, oldname, newname, safe=False): + if self.readonly: + raise ReadOnlyError + + if self.file_exists(self._fpath(newname)): + if safe: + raise NameError("File %r exists" % newname) + else: + self.delete_file(self._fpath(newname)) + key = media_storage.key_class(bucket=media_storage.bucket, name=self._fpath(oldname)) + key.copy(media_storage.bucket, self._fpath(newname)) + key.delete() + + def lock(self, name): + if name not in self.locks: + self.locks[name] = Lock() + return self.locks[name] + + def temp_storage(self, name=None): + name = name or "%s.tmp" % random_name() + path = os.path.join(self.folder, name) + tempstore = S3FileStorage(path) + return tempstore.create() + + +class S3WhooshSearchBackend(WhooshSearchBackend): + def setup(self): + """ + Defers loading until needed. + """ + from haystack import connections + new_index = False + + # Make sure the index is there. + if not media_storage.exists(self.path): + new_index = True + + self.storage = S3FileStorage(self.path) + + self.content_field_name, self.schema = self.build_schema( + connections[self.connection_alias].get_unified_index().all_searchfields()) + self.parser = QueryParser(self.content_field_name, schema=self.schema) + + if new_index is True: + self.index = self.storage.create_index(self.schema) + else: + try: + self.index = self.storage.open_index(schema=self.schema) + except index.EmptyIndexError: + self.index = self.storage.create_index(self.schema) + + self.setup_complete = True + + +class S3WhooshEngine(WhooshEngine): + backend = S3WhooshSearchBackend