diff --git a/bin/whisper-resize.py b/bin/whisper-resize.py index 501e5f69..9499a8bf 100755 --- a/bin/whisper-resize.py +++ b/bin/whisper-resize.py @@ -110,7 +110,7 @@ for archive in old_archives: # Loading all datapoints into memory for fast querying timeinfo, values = archive['data'] - new_datapoints = zip(range(*timeinfo), values) + new_datapoints = list(zip(range(*timeinfo), values)) if all_datapoints: last_timestamp = all_datapoints[-1][0] slice_end = 0 @@ -122,8 +122,8 @@ else: all_datapoints += new_datapoints - oldtimestamps = map(lambda p: p[0], all_datapoints) - oldvalues = map(lambda p: p[1], all_datapoints) + oldtimestamps = list(map(lambda p: p[0], all_datapoints)) + oldvalues = list(map(lambda p: p[1], all_datapoints)) print("oldtimestamps: %s" % oldtimestamps) # Simply cleaning up some used memory @@ -148,8 +148,8 @@ righti = bisect.bisect_left(oldtimestamps, tinterval[1], lo=lefti) newvalues = oldvalues[lefti:righti] if newvalues: - non_none = filter(lambda x: x is not None, newvalues) - if 1.0 * len(non_none) / len(newvalues) >= xff: + non_none = list(filter(lambda x: x is not None, newvalues)) + if non_none and 1.0 * len(non_none) / len(newvalues) >= xff: newdatapoints.append([tinterval[0], whisper.aggregate(aggregationMethod, non_none, newvalues)]) diff --git a/contrib/update-storage-times.py b/contrib/update-storage-times.py index 218a04da..5ae970e2 100755 --- a/contrib/update-storage-times.py +++ b/contrib/update-storage-times.py @@ -21,8 +21,6 @@ except ImportError: from os import listdir as scandir -RESIZE_BIN = "/opt/graphite/bin/whisper-resize.py" -INFO_BIN = "/opt/graphite/bin/whisper-info.py" LOG = logging.getLogger() LOG.setLevel(logging.INFO) SCHEMA_LIST = {} @@ -31,7 +29,6 @@ 'retentions': '1m:7d'} DEBUG = False DRY_RUN = False -BASE_COMMAND = [RESIZE_BIN] ROOT_PATH = "" @@ -92,7 +89,7 @@ def fix_metric(metric): command_string = list(BASE_COMMAND) + [metric] retention = DEFAULT_SCHEMA['retentions'] - matching = metric[len(ROOT_PATH):] + matching = metric[len(ROOT_PATH):].replace('/', '.') for schema, info in SCHEMA_LIST.iteritems(): if info['match'].search(matching): retention = info['retentions'] @@ -109,14 +106,19 @@ def fix_metric(metric): res = 0 else: LOG.debug('Retention will be %s' % retention) + # record file owner/group and perms to set properly after whisper-resize.py is complete + st = os.stat(metric) if DEBUG: res = subprocess.check_call(command_string) else: res = subprocess.check_call(command_string, stdout=devnull) + os.chmod(metric, st.st_mode) + os.chown(metric, st.st_uid, st.st_gid) + devnull.close() # wait for a second, so we don't kill I/O on the host - time.sleep(0.3) + time.sleep(SLEEP) """ We have manual commands for every failed file from these errors, so we can just go through each of these errors @@ -170,6 +172,12 @@ def cli_opts(): parser.add_argument('--aggregate', action='store_true', dest='aggregate', help="Passed through to whisper-resize.py, roll up values", default=False) + parser.add_argument('--bindir', action='store', dest='bindir', + help="The root path to whisper-resize.py and whisper-info.py", + default='/opt/graphite/bin') + parser.add_argument('--sleep', action='store', dest='sleep', + help="Sleep this amount of time in seconds between metric comparisons", + default=0.3) return parser.parse_args() @@ -187,6 +195,12 @@ def cli_opts(): ROOT_PATH = i_args.path DEBUG = i_args.debug DRY_RUN = i_args.dry_run + BINDIR = i_args.bindir + SLEEP = i_args.sleep + RESIZE_BIN = BINDIR + "/whisper-resize.py" + INFO_BIN = BINDIR + "/whisper-info.py" + BASE_COMMAND = [RESIZE_BIN] + if i_args.nobackup: BASE_COMMAND.append('--nobackup') if i_args.aggregate: