Merge pull request #168 from jazzl0ver/master

better implementation of cache_path variable access for fixing #150 
multiple other fixes

README.md

@@ -141,7 +141,7 @@ Install FUSE for OS X from <http://osxfuse.github.com>.
 
     # Put contrib/mount.yas3fs to /usr/local/sbin and make the symlink
     chmod +x /usr/local/sbin/mount.yas3fs
-    cd /sbin; sudo ln -s /usr/local/sbin/mount.yas3fs
+    cd /sbin; sudo ln -s /usr/local/sbin/mount.yas3fs.centos6 # replace centos6 to amzn1 for Amazon Linux installation
     # Add the contents of contrib/fstab.snippet to /etc/fstab and modify accordingly
     # Try to mount
     mount /mnt/mybucket

contrib/fstab.snippet

@@ -1 +1 @@
-yas3fs#mybucket /mnt/mybucket yas3fs _netdev,allow_other,default_permissions,topic="arn:aws:sns:us-east-1:0000000000:mybucket",queue=yas3fs-mybucket,yas3fslog,yas3fsdebug 0 0
+yas3fs#mybucket /mnt/mybucket yas3fs _netdev,allow_other,default_permissions,topic=arn:aws:sns:us-east-1:0000000000:mybucket,yas3fslog,yas3fsdebug 0 0

contrib/mount.yas3fs.amzn1

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+s3bucket=${1##*#}       # extract the last part after '#' in 'yas3fs#bucket' string
+localpath=$2
+topic=$(echo $4 | /bin/grep -o 'topic=[^,]*' | /bin/cut -f2 -d=)
+queue=$(echo $4 | /bin/grep -o 'queue=[^,]*' | /bin/cut -f2 -d=)
+log=$(echo $4 | /bin/grep -o 'yas3fslog')
+debug=$(echo $4 | /bin/grep -o 'yas3fsdebug')
+logline=""
+
+if [ "x$log" = "xyas3fslog" ]; then
+    logline="--log /tmp/.yas3fs-${s3bucket} --log-mb-size 10 --log-backup-count 10 --log-backup-gzip"
+    [ "x$debug" = "xyas3fsdebug" ] && logline="-d $logline"
+fi
+
+topicline=""
+queueline=""
+if [ ! "x$topic" = "x" ]; then
+    topicline="--topic ${topic}"
+    if [ "x$queue" = "x" ]; then
+        queueline="--new-queue-with-hostname"
+    else
+        queueline="--queue ${queue}"
+    fi
+fi
+
+yas3fs --download-retries-num 10 --recheck-s3 ${logline} "s3://${s3bucket}" "${localpath}" ${topicline} ${queueline}

contrib/mount.yas3fs.centos6

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+s3bucket=${1##*#}       # extract the last part after '#' in 'yas3fs#bucket' string
+localpath=$2
+topic=$(echo $4 | /bin/grep -o 'topic=[^,]*' | /bin/cut -f2 -d=)
+queue=$(echo $4 | /bin/grep -o 'queue=[^,]*' | /bin/cut -f2 -d=)
+log=$(echo $4 | /bin/grep -o 'yas3fslog')
+debug=$(echo $4 | /bin/grep -o 'yas3fsdebug')
+logline=""
+
+if [ "x$log" = "xyas3fslog" ]; then
+    logline="--log /tmp/.yas3fs-${s3bucket} --log-mb-size 10 --log-backup-count 10 --log-backup-gzip"
+    [ "x$debug" = "xyas3fsdebug" ] && logline="-d $logline"
+fi
+
+topicline=""
+queueline=""
+if [ ! "x$topic" = "x" ]; then
+    topicline="--topic ${topic}"
+    if [ "x$queue" = "x" ]; then
+        queueline="--new-queue-with-hostname"
+    else
+        queueline="--queue ${queue}"
+    fi
+fi
+
+scl enable python27 -- yas3fs --download-retries-num 10 --recheck-s3 ${logline} "s3://${s3bucket}" "${localpath}" ${topicline} ${queueline}

yas3fs/__init__.py

@@ -31,7 +31,7 @@
 import datetime as dt
 import gc # For debug only
 import pprint # For debug only
-import tempfile
+from tempfile import mkdtemp
 from shutil import rmtree
 
 if sys.version_info < (3, ):  # python2
@@ -863,7 +863,7 @@ def __init__(self, options):
             cache_path_prefix = 'yas3fs-' + self.s3_bucket_name + '-'
             if not self.s3_prefix == '':
                 cache_path_prefix += self.s3_prefix + '-'
-        self.cache_path = tempfile.mkdtemp(prefix = cache_path_prefix)
+        self.cache_path = mkdtemp(prefix = cache_path_prefix)
         logger.info("Cache path (on disk): '%s'" % self.cache_path)
         self.cache = FSCache(self.cache_path)
         self.publish_queue = Queue()
@@ -1024,6 +1024,10 @@ def __init__(self, options):
         if self.plugin:
             self.plugin.logger = logger
 
+        # save this object for later use in remove_empty_dirs()
+        global yas3fsobj
+        yas3fsobj = self
+
         signal.signal(signal.SIGINT, self.signal_handler)
         signal.signal(signal.SIGHUP, self.signal_handler)
 
@@ -1287,8 +1291,8 @@ def process_message(self, messages):
                     self.invalidate_cache(path)
         elif c[1] == 'md':
             if c[2]:
-                self.cache.delete(c[3], 'key')
-                self.cache.delete(c[3], c[2])
+        	self.delete_cache(c[2])
+        	self.delete_cache(c[3])
         elif c[1] == 'reset':
             if len(c) <= 2 or not c[2] or c[2] == '/':
                 with self.cache.lock:
@@ -1925,7 +1929,15 @@ def symlink(self, path, link):
     def check_data(self, path):
         logger.debug("check_data '%s'" % (path))
         with self.cache.get_lock(path):
-            data = self.cache.get(path, 'data')
+            #-- jazzl0ver: had to add path checking due to untracable /by me/ cache leaking (workaround for issue #174)
+    	    data = self.cache.get(path, 'data')
+            if data and not os.path.exists(self.cache.get_cache_filename(path)):
+        	logger.debug("Cache leak found for '%s', cleaning up..." % (path))
+        	self.cache.delete(path)
+                with self.cache.lock and self.cache.new_locks[path]:
+                    del self.cache.new_locks[path]
+        	del self.cache.unused_locks[path]
+    		data = self.cache.get(path, 'data')
             if not data or data.has('new'):
                 k = self.get_key(path)
                 if not k:
@@ -3102,13 +3114,9 @@ def remove_empty_dirs(dirname):
             dirname = dirname.encode('utf-8')
 
         # fix for https://github.com/danilop/yas3fs/issues/150
-        # probably not the best way to find the cache_path value
-        for obj in gc.get_objects():
-            if isinstance(obj, YAS3FS):
-                cache_path = obj.cache_path
-                # remove cache_path part from dirname to avoid accidental removal of /tmp (if empty)
-                os.chdir(cache_path)
-                dirname = dirname.replace(cache_path + '/', '')
+        # remove cache_path part from dirname to avoid accidental removal of /tmp (if empty)
+        os.chdir(yas3fsobj.cache_path)
+        dirname = dirname.replace(yas3fsobj.cache_path + '/', '')
 
         os.removedirs(dirname)
         logger.debug("remove_empty_dirs '%s' done" % (dirname))

yas3fs/_version.py

@@ -1 +1 @@
-__version__ = '2.4.0'
+__version__ = '2.4.4'