Merge pull request #270 from deniszh/backport/1.1.x/pr-265_pr-265_pr-…

…265_pr-265_pr-265_pr-265_pr-265_pr-266_pr-269

[1.1.x] refactored update-storage-times.py to allow alternate whisper-resize.py and whisper-info.py location, modified matching to convert filesystem path to graphite dot-style used in storage-schemas | recor

bin/whisper-resize.py

@@ -110,7 +110,7 @@
   for archive in old_archives:
     # Loading all datapoints into memory for fast querying
     timeinfo, values = archive['data']
-    new_datapoints = zip(range(*timeinfo), values)
+    new_datapoints = list(zip(range(*timeinfo), values))
     if all_datapoints:
       last_timestamp = all_datapoints[-1][0]
       slice_end = 0
@@ -122,8 +122,8 @@
     else:
       all_datapoints += new_datapoints
 
-  oldtimestamps = map(lambda p: p[0], all_datapoints)
-  oldvalues = map(lambda p: p[1], all_datapoints)
+  oldtimestamps = list(map(lambda p: p[0], all_datapoints))
+  oldvalues = list(map(lambda p: p[1], all_datapoints))
 
   print("oldtimestamps: %s" % oldtimestamps)
   # Simply cleaning up some used memory
@@ -148,8 +148,8 @@
       righti = bisect.bisect_left(oldtimestamps, tinterval[1], lo=lefti)
       newvalues = oldvalues[lefti:righti]
       if newvalues:
-        non_none = filter(lambda x: x is not None, newvalues)
-        if 1.0 * len(non_none) / len(newvalues) >= xff:
+        non_none = list(filter(lambda x: x is not None, newvalues))
+        if non_none and 1.0 * len(non_none) / len(newvalues) >= xff:
           newdatapoints.append([tinterval[0],
                                 whisper.aggregate(aggregationMethod,
                                                   non_none, newvalues)])

contrib/update-storage-times.py

@@ -21,8 +21,6 @@
 except ImportError:
     from os import listdir as scandir
 
-RESIZE_BIN = "/opt/graphite/bin/whisper-resize.py"
-INFO_BIN = "/opt/graphite/bin/whisper-info.py"
 LOG = logging.getLogger()
 LOG.setLevel(logging.INFO)
 SCHEMA_LIST = {}
@@ -31,7 +29,6 @@
                   'retentions': '1m:7d'}
 DEBUG = False
 DRY_RUN = False
-BASE_COMMAND = [RESIZE_BIN]
 ROOT_PATH = ""
 
 
@@ -92,7 +89,7 @@ def fix_metric(metric):
     command_string = list(BASE_COMMAND) + [metric]
 
     retention = DEFAULT_SCHEMA['retentions']
-    matching = metric[len(ROOT_PATH):]
+    matching = metric[len(ROOT_PATH):].replace('/', '.')
     for schema, info in SCHEMA_LIST.iteritems():
         if info['match'].search(matching):
             retention = info['retentions']
@@ -109,14 +106,19 @@ def fix_metric(metric):
         res = 0
     else:
         LOG.debug('Retention will be %s' % retention)
+        # record file owner/group and perms to set properly after whisper-resize.py is complete
+        st = os.stat(metric)
         if DEBUG:
             res = subprocess.check_call(command_string)
         else:
             res = subprocess.check_call(command_string,
                                         stdout=devnull)
+        os.chmod(metric, st.st_mode)
+        os.chown(metric, st.st_uid, st.st_gid)
+
     devnull.close()
     # wait for a second, so we don't kill I/O on the host
-    time.sleep(0.3)
+    time.sleep(SLEEP)
     """
     We have manual commands for every failed file from these
     errors, so we can just go through each of these errors
@@ -170,6 +172,12 @@ def cli_opts():
     parser.add_argument('--aggregate', action='store_true', dest='aggregate',
                         help="Passed through to whisper-resize.py, roll up values",
                         default=False)
+    parser.add_argument('--bindir', action='store', dest='bindir',
+                        help="The root path to whisper-resize.py and whisper-info.py",
+                        default='/opt/graphite/bin')
+    parser.add_argument('--sleep', action='store', dest='sleep',
+                        help="Sleep this amount of time in seconds between metric comparisons",
+                        default=0.3)
     return parser.parse_args()
 
 
@@ -187,6 +195,12 @@ def cli_opts():
     ROOT_PATH = i_args.path
     DEBUG = i_args.debug
     DRY_RUN = i_args.dry_run
+    BINDIR = i_args.bindir
+    SLEEP = i_args.sleep
+    RESIZE_BIN = BINDIR + "/whisper-resize.py"
+    INFO_BIN = BINDIR + "/whisper-info.py"
+    BASE_COMMAND = [RESIZE_BIN]
+
     if i_args.nobackup:
         BASE_COMMAND.append('--nobackup')
     if i_args.aggregate: