Merge pull request #167 from jazzl0ver/master

Please, incorporate my changes

README.md

@@ -72,6 +72,8 @@ Install using [pip](http://www.pip-installer.org/en/latest/).
 
     pip install yas3fs
 
+If it fails, check the CentOS 6 installation steps below.
+
 If you want to do a quick test here's the installation procedure depending on the OS flavor (Linux or Mac):
 
 * Create an S3 bucket in the AWS region you prefer.
@@ -118,6 +120,32 @@ Install FUSE for OS X from <http://osxfuse.github.com>.
     # For multiple hosts mount
     yas3fs s3://BUCKET/PATH LOCAL-PATH --topic TOPIC-ARN --new-queue
 
+**On CentOS 6**
+
+    sudo yum -y install fuse fuse-libs centos-release-scl
+    sudo yum -y install python27
+    # upgrade setuptools
+    scl enable python27 -- pip install setuptools --upgrade
+    # grab the latest sources
+    git clone https://github.com/danilop/yas3fs.git
+    cd yas3fs
+    scl enable python27 -- python setup.py install
+    scl enable python27 -- yas3fs -h # See the usage
+    mkdir LOCAL-PATH
+    # For single host mount
+    scl enable python27 -- yas3fs s3://BUCKET/PATH LOCAL-PATH
+    # For multiple hosts mount
+    scl enable python27 -- yas3fs s3://BUCKET/PATH LOCAL-PATH --topic TOPIC-ARN --new-queue
+
+**/etc/fstab support**
+
+    # Put contrib/mount.yas3fs to /usr/local/sbin and make the symlink
+    chmod +x /usr/local/sbin/mount.yas3fs
+    cd /sbin; sudo ln -s /usr/local/sbin/mount.yas3fs
+    # Add the contents of contrib/fstab.snippet to /etc/fstab and modify accordingly
+    # Try to mount
+    mount /mnt/mybucket
+
 To listen to SNS HTTP notifications (I usually suggest to use SQS instead) with a Mac
 you need to install the Python [M2Crypto](http://chandlerproject.org/Projects/MeTooCrypto) module,
 download the most suitable "egg" from
@@ -132,9 +160,17 @@ If something does not work as expected you can use the `-df` options to run in f
 To unmount the file system on Linux:
 
     fusermount -u LOCAL-PATH
+    or
+    umount LOCAL-PATH
+
+The latter works if /etc/fstab support steps (see above) were completed
 
 To unmount the file system on a Mac you can use `umount`.
 
+**rsync usage**
+
+rsync's option *--inplace* has to be used to avoid S3 busy events
+
 ### Full Usage
 
     yas3fs -h
@@ -446,6 +482,7 @@ This may be useful to anyone who wants to validate/test YAS3FS to see how it beh
 {
   "Effect": "Allow",
   "Action": [
+      "s3:GetBucketLocation",
       "s3:DeleteObject",
       "s3:GetObject",
       "s3:GetObjectVersion",
@@ -492,5 +529,14 @@ This may be useful to anyone who wants to validate/test YAS3FS to see how it beh
   ]
 }
 ```
-
+##### IAM
+```JSON
+{
+  "Effect": "Allow",
+  "Action": "iam:GetUser",
+  "Resource": [
+      "*"
+  ]
+}
+```
 Happy File Sharing!

contrib/fstab.snippet

@@ -0,0 +1 @@
+yas3fs#mybucket /mnt/mybucket yas3fs _netdev,allow_other,default_permissions,topic="arn:aws:sns:us-east-1:0000000000:mybucket",queue=yas3fs-mybucket,yas3fslog,yas3fsdebug 0 0

contrib/mount.yas3fs

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+s3bucket=${1##*#}       # extract the last part after '#' in 'yas3fs#bucket' string
+localpath=$2
+topic=$(echo $4 | /bin/grep -o 'topic=[^,]*' | /bin/cut -f2 -d=)
+queue=$(echo $4 | /bin/grep -o 'queue=[^,]*' | /bin/cut -f2 -d=)
+log=$(echo $4 | /bin/grep -o 'yas3fslog')
+debug=$(echo $4 | /bin/grep -o 'yas3fsdebug')
+logline=""
+
+if [ "x$log" = "xyas3fslog" ]; then
+    logline="--log \"/tmp/.yas3fs-${s3bucket}\" --log-mb-size 1 --log-backup-count 3 --log-backup-gzip"
+    [ "x$debug" = "xyas3fsdebug" ] && log="-d $log"
+fi
+
+scl enable python27 -- yas3fs --download-retries-num 10 ${logline} "s3://${s3bucket}" "${localpath}" --topic "${topic}" --queue "${queue}"

setup.py

@@ -4,7 +4,7 @@
 
 exec(open('yas3fs/_version.py').read())
 
-requires = ['setuptools>=2.2', 'boto>=2.25.0']
+requires = ['setuptools>=2.2', 'boto>=2.25.0', 'boto3>=1.6.12']
 
 # Versions of Python pre-2.7 require argparse separately. 2.7+ and 3+ all
 # include this as the replacement for optparse.

yas3fs/__init__.py

@@ -31,6 +31,8 @@
 import datetime as dt
 import gc # For debug only
 import pprint # For debug only
+import tempfile
+from shutil import rmtree
 
 if sys.version_info < (3, ):  # python2
     from urllib import unquote_plus
@@ -62,6 +64,8 @@
 from boto.utils import compute_md5, compute_hash
 from boto.s3.key import Key
 
+import boto3
+
 from .YAS3FSPlugin import YAS3FSPlugin
 
 from ._version import __version__
@@ -854,13 +858,14 @@ def __init__(self, options):
 
         # Internal Initialization
         if options.cache_path:
-            cache_path = options.cache_path
+            cache_path_prefix = options.cache_path
         else:
-            cache_path = '/tmp/yas3fs/' + self.s3_bucket_name
+            cache_path_prefix = 'yas3fs-' + self.s3_bucket_name + '-'
             if not self.s3_prefix == '':
-                cache_path += '/' + self.s3_prefix
-        logger.info("Cache path (on disk): '%s'" % cache_path)
-        self.cache = FSCache(cache_path)
+                cache_path_prefix += self.s3_prefix + '-'
+        self.cache_path = tempfile.mkdtemp(prefix = cache_path_prefix)
+        logger.info("Cache path (on disk): '%s'" % self.cache_path)
+        self.cache = FSCache(self.cache_path)
         self.publish_queue = Queue()
         self.s3_queue = {} # Of Queue()
         for i in range(self.s3_num):
@@ -980,13 +985,18 @@ def __init__(self, options):
             logger.info("SQS queue name (new): '%s'" % self.sqs_queue_name)
             self.queue.set_message_class(boto.sqs.message.RawMessage) # There is a bug with the default Message class in boto
 
-            self.current_user_aws_principalId = None
+            self.current_user_principalId = None
             try:
                 iam = boto.connect_iam()
                 self.current_user_principalId = 'AWS:'+iam.get_user()['get_user_response']['get_user_result']['user']['user_id']
                 logger.info("Current user principalId: "+self.current_user_principalId)
             except Exception as e:
-                logger.warn("Failed to get current user principalId: "+str(e))
+               try:
+                   sts = boto3.client('sts')
+                   self.current_user_principalId = 'AWS:'+sts.get_caller_identity()['UserId']
+                   logger.info("Current user principalId: "+self.current_user_principalId)
+               except Exception as e:
+                   logger.warn("Failed to get current user principalId: "+str(e))
 
         if self.hostname or self.sns_http_port:
             if not self.sns_topic_arn:
@@ -1182,6 +1192,8 @@ def destroy(self, path):
             self.cache_entries = 0 # To stop memory thread
             logger.info("waiting for check cache thread to shutdown...")
             self.check_cache_thread.join(self.cache_check_interval + 1.0)
+            logger.info("deleting cache_path %s ..." % self.cache_path)
+            rmtree(self.cache_path)
         logger.info('File system unmounted.')
 
     def listen_for_messages_over_http(self):
@@ -3089,6 +3101,15 @@ def remove_empty_dirs(dirname):
         if not isinstance(dirname, bytes):
             dirname = dirname.encode('utf-8')
 
+        # fix for https://github.com/danilop/yas3fs/issues/150
+        # probably not the best way to find the cache_path value
+        for obj in gc.get_objects():
+            if isinstance(obj, YAS3FS):
+                cache_path = obj.cache_path
+                # remove cache_path part from dirname to avoid accidental removal of /tmp (if empty)
+                os.chdir(cache_path)
+                dirname = dirname.replace(cache_path + '/', '')
+
         os.removedirs(dirname)
         logger.debug("remove_empty_dirs '%s' done" % (dirname))
     except OSError as exc: # Python >2.5
@@ -3195,7 +3216,7 @@ def main():
     parser.add_argument('--cache-disk-size', metavar='N', type=int, default=1024,
                         help='max size of the disk cache in MB (default is %(default)s MB)')
     parser.add_argument('--cache-path', metavar='PATH', default='',
-                        help='local path to use for disk cache (default is /tmp/yas3fs/BUCKET/PATH)')
+                        help='local path to use for disk cache (default is /tmp/yas3fs-BUCKET-PATH-random)')
     parser.add_argument('--recheck-s3', action='store_true',
                         help='Cached ENOENT (error no entry) rechecks S3 for new file/directory')
     parser.add_argument('--cache-on-disk', metavar='N', type=int, default=0,
@@ -3266,6 +3287,8 @@ def main():
                         help='requester pays for S3 interactions, the bucket must have Requester Pays enabled')
     parser.add_argument('--no-allow-other', action='store_true',
                         help='Do not allow other users to access mounted directory')
+    parser.add_argument('--no-default-permissions', action='store_true',
+                        help='do NOT honor file system permissions for non-root users')
     parser.add_argument('--with-plugin-file', metavar='FILE',
                         help="YAS3FSPlugin file")
     parser.add_argument('--with-plugin-class', metavar='CLASS',
@@ -3330,7 +3353,8 @@ def main():
         'max_read':131072,
         'max_write':131072,
         'max_readahead':131072,
-        'direct_io':True
+        'direct_io':True,
+        'default_permissions':True
         }
     if options.no_allow_other:
         mount_options["allow_other"] = False
@@ -3342,6 +3366,8 @@ def main():
         mount_options['umask'] = options.umask
     if options.read_only:
         mount_options['ro'] = True
+    if options.no_default_permissions:
+        mount_options["default_permissions"] = False
 
     if options.nonempty:
         mount_options['nonempty'] = True

yas3fs/_version.py

@@ -1 +1 @@
-__version__ = '2.3.5'
+__version__ = '2.4.0'