test file store

settings.py

@@ -62,14 +62,14 @@ HAYSTACK_CUSTOM_HIGHLIGHTER = 'project.search.Highlighter'
 HAYSTACK_DEFAULT_OPERATOR = 'OR'
 HAYSTACK_CONNECTIONS = {
     'default': {
-        'ENGINE': 'haystack.backends.whoosh_backend.WhooshEngine',
+        'ENGINE': 'project.filestore.S3WhooshEngine',
         'PATH': os.path.join('/whoosh', 'default'),
     },
 }
 for lang_code, lang in LANGUAGES:
     HAYSTACK_CONNECTIONS[lang_code] = {}
     HAYSTACK_CONNECTIONS[lang_code].update({
-        'ENGINE': 'haystack.backends.whoosh_backend.WhooshEngine',
+        'ENGINE': 'project.filestore.S3WhooshEngine',
         'PATH': os.path.join('/whoosh', lang_code),
     })
 

src/project/filestore.py

@@ -0,0 +1,133 @@
+from threading import Lock
+
+from aldryn_django.storage import S3MediaStorage
+from haystack.backends.whoosh_backend import WhooshEngine, WhooshSearchBackend
+from whoosh import index
+from whoosh.filedb.filestore import Storage, ReadOnlyError
+
+import os
+from whoosh.filedb.structfile import StructFile
+from whoosh.qparser import QueryParser
+from whoosh.util import random_name
+
+media_storage = S3MediaStorage()
+
+
+class S3FileStorage(Storage):
+    supports_mmap = False
+
+    def __init__(self, path, readonly=False, debug=False):
+        self.folder = path
+        self.readonly = readonly
+        self._debug = debug
+        self.locks = {}
+
+    def __repr__(self):
+        return "%s(%r)" % (self.__class__.__name__, self.folder)
+
+    def create(self):
+        return self
+
+    def destroy(self):
+        del self.locks
+        self.clean()
+
+    def create_file(self, name, excl=False, mode="wb", **kwargs):
+        if self.readonly:
+            raise ReadOnlyError
+
+        path = self._fpath(name)
+        f = StructFile(media_storage.open(path, mode=mode), name=name, **kwargs)
+        return f
+
+    def open_file(self, name, **kwargs):
+        path = self._fpath(name)
+        f = StructFile(media_storage.open(path, 'rb'), name=name, **kwargs)
+        return f
+
+    def _fpath(self, fname):
+        return os.path.abspath(os.path.join(self.folder, fname))
+
+    def clean(self, ignore=False):
+        if self.readonly:
+            raise ReadOnlyError
+
+        files = self.list()
+        for name in files:
+            self.delete_file(name)
+
+    def list(self):
+        dirs, files = media_storage.listdir(self.folder)
+        return files
+
+    def file_exists(self, name):
+        return media_storage.exists(self._fpath(name))
+
+    def file_modified(self, name):
+        return media_storage.get_modified_time(self._fpath(name))
+
+    def file_length(self, name):
+        return media_storage.size(self._fpath(name))
+
+    def delete_file(self, name):
+        if self.readonly:
+            raise ReadOnlyError
+
+        media_storage.delete(self._fpath(name))
+
+    def rename_file(self, oldname, newname, safe=False):
+        if self.readonly:
+            raise ReadOnlyError
+
+        if self.file_exists(self._fpath(newname)):
+            if safe:
+                raise NameError("File %r exists" % newname)
+            else:
+                self.delete_file(self._fpath(newname))
+        key = media_storage.key_class(bucket=media_storage.bucket, name=self._fpath(oldname))
+        key.copy(media_storage.bucket, self._fpath(newname))
+        key.delete()
+
+    def lock(self, name):
+        if name not in self.locks:
+            self.locks[name] = Lock()
+        return self.locks[name]
+
+    def temp_storage(self, name=None):
+        name = name or "%s.tmp" % random_name()
+        path = os.path.join(self.folder, name)
+        tempstore = S3FileStorage(path)
+        return tempstore.create()
+
+
+class S3WhooshSearchBackend(WhooshSearchBackend):
+    def setup(self):
+        """
+        Defers loading until needed.
+        """
+        from haystack import connections
+        new_index = False
+
+        # Make sure the index is there.
+        if not media_storage.exists(self.path):
+            new_index = True
+
+        self.storage = S3FileStorage(self.path)
+
+        self.content_field_name, self.schema = self.build_schema(
+            connections[self.connection_alias].get_unified_index().all_searchfields())
+        self.parser = QueryParser(self.content_field_name, schema=self.schema)
+
+        if new_index is True:
+            self.index = self.storage.create_index(self.schema)
+        else:
+            try:
+                self.index = self.storage.open_index(schema=self.schema)
+            except index.EmptyIndexError:
+                self.index = self.storage.create_index(self.schema)
+
+        self.setup_complete = True
+
+
+class S3WhooshEngine(WhooshEngine):
+    backend = S3WhooshSearchBackend