Skip records (#79)

* new options

* new options

* process simu

* skip records and delete records

* debug info better

* debug

* default rate increase for se

* ok for failing cuts

* ok for failing cuts

* no more config

* fixed bad logic in delete_records

* avoid overwrite

* final for sr1

.gitignore

@@ -1,3 +1,6 @@
+# configs
+jobs/config.ini
+
 # Generated
 generated/
 

jobs/config.ini

@@ -4,29 +4,28 @@ t_sleep = 1
 
 [job]
 container = xenonnt-2024.02.1.simg
-#runids = 021952,022510,022776,022773,022738,022735,022732,022726,022723,022717,022694,022691,022685,022682,022679,022676,022670,022667,028636,031661,031124,029866,027982,030329,025444,027513,030891,025416,025161,028945,031709,028239,030937,028510,028636,031661,031124,029866,027982,030329,025444,027513,030891,025416,025161,028945,031709,028239,030937,028510,027323,026550,025953,031657,029198,026577,028031,025719,029208,026624,031147,028426,030102,029144,029472,027705,031156,031802,026503,029139,028047,025579,030079,025953,028542,025413,031803,029938,030287,026603,028890,030128,029410,031761
-#runids = 020298,020299,020300,020301,020302,020303,020304,020305,020306,020307,020308,020309,020310,020311,020312,020313,020314,020315,020316,020317,020318,020319,020320,020321,020322,020323,020324,020325
 # ambe and rn220 available
 #runids = 020298,020303,020304,022510,022735,022723,022717,022685
-runids = 051613,050996,051948,051939,051910,051909,051905,047876,047865,047860,053167,053153,053139,052997,049794,048965,048880,049123,049450,049505,049521,049544
-output_folder = /scratch/midway2/yuanlq/salt/beta
+# sr1
+runids = 051613,050996,051948,051939,051910,051909,051905,047876,047865,047860,053167,053153,053139,052997,049123,049450,049505,049521,049544
+output_folder = /scratch/midway2/yuanlq/salt/se
 saltax_mode = salt
 faxconf_version = sr0_v4
-generator_name = flat
+generator_name = se
 recoil = 8
 mode = all
 process_data = False
 process_simu = True
-skip_records = True
+skip_records = False
 delete_records = True
 storage_to_patch = /project/lgrandi/yuanlq/salt/raw_records,/scratch/midway3/yuanlq/salt/raw_records
 
 [slurm]
 username = yuanlq
 account = pi-lgrandi
 job_title = salt
-partition = lgrandi
-qos = lgrandi
-mem_per_cpu = 45000
+partition = caslake
+qos = caslake
+mem_per_cpu = 70000
 cpus_per_task = 1
-log_dir = /home/yuanlq/.tmp_job_submission/saltax/beta
+log_dir = /home/yuanlq/.tmp_job_submission/saltax/se

jobs/job.py

@@ -55,25 +55,18 @@
 gc.collect()
 for dt in to_process_dtypes:
 	print("Making %s. "%dt)
-	st.make(strrunid, dt, save=(dt))
-	print("Done with %s. "%dt)
+	try:
+		st.make(strrunid, dt, save=(dt))
+		print("Done with %s. "%dt)
+	except NotImplementedError as e:
+		print("The cut_basics for run %d is not implemented. "%runid)
 	gc.collect()
 
 print("Used time:", datetime.now() - now)
 now = datetime.now()
 
 print("Finished making all the computation for run %d in \
 	saltax mode salt. "%(runid))
-if delete_records:
-	print("Deleting records.")
-	records_name = str(st.key_for(strrunid, 'records'))
-	records_path = os.path.join(output_folder, records_name)
-	if os.path.exists(records_path):
-		os.rmdir(records_path)
-		gc.collect()
-		print("Deleted records for run %d in saltax mode salt. "%(runid))
-print('====================')
-
 
 if saltax_mode == 'salt':
 	if process_data:
@@ -93,8 +86,11 @@
 
 		for dt in to_process_dtypes:
 			print("Making %s. "%dt)
-			st.make(strrunid, dt, save=(dt))
-			print("Done with %s. "%dt)
+			try:
+				st.make(strrunid, dt, save=(dt))
+				print("Done with %s. "%dt)
+			except NotImplementedError as e:
+				print("The cut_basics for run %d is not implemented. "%runid)
 			gc.collect()
 
 		print("Used time:", datetime.now() - now)
@@ -105,14 +101,7 @@
 
 		print("Finished making all the computation for run %d in \
 			saltax mode data. "%(runid))
-		if delete_records:
-			print("Deleting records.")
-			records_name = str(st.key_for(strrunid, 'records'))
-			records_path = os.path.join(output_folder, records_name)
-			if os.path.exists(records_path):
-				os.rmdir(records_path)
-				gc.collect()
-				print("Deleted records for run %d in saltax mode data. "%(runid))
+
 		print('====================')
 	else:
 		print("You specified process_data = False, so we will not process data.")
@@ -135,8 +124,12 @@
 		gc.collect()
 		for dt in to_process_dtypes:
 			print("Making %s. "%dt)
-			st.make(strrunid, dt, save=(dt))
-			print("Done with %s. "%dt)
+
+			try:
+				st.make(strrunid, dt, save=(dt))
+				print("Done with %s. "%dt)
+			except NotImplementedError as e:
+				print("The cut_basics for run %d is not implemented. "%runid)
 			gc.collect()
 
 		print("Used time:", datetime.now() - now)
@@ -147,16 +140,19 @@
 
 		print("Finished making all the computation for run %d in \
 			saltax mode simu. "%(runid))
-		if delete_records:
-			print("Deleting records.")
-			records_name = str(st.key_for(strrunid, 'records'))
-			records_path = os.path.join(output_folder, records_name)
-			if os.path.exists(records_path):
-				os.rmdir(records_path)
-				gc.collect()
-				print("Deleted records for run %d in saltax mode simu. "%(runid))
 		print('====================')
 	else:
 		print("You specified process_simu = False, so we will not process simu.")
 
+if delete_records:
+	print("Deleting records.")
+	records_name = str(st.key_for(strrunid, 'records'))
+	records_path = os.path.join(output_folder, records_name)
+	if os.path.exists(records_path):
+		os.rmdir(records_path)
+		gc.collect()
+		print("Deleted records for run %d in saltax mode salt. "%(runid))
+print('====================')
+
+
 print("Finished all. Exiting.")

saltax/instructions/generator.py

@@ -151,7 +151,12 @@ def instr_file_name(runid, instr, recoil, generator_name, mode, rate=1e9/SALT_TI
     filename = BASE_DIR + runid + "-" + str(recoil) + "-" + \
         generator_name + "-" + mode + "-" + str(rate) + ".csv"
 
-    pd.DataFrame(instr).to_csv(filename, index=False)
+    # if the file already exists, we don't want to overwrite it
+    if not os.path.exists(filename):
+        pd.DataFrame(instr).to_csv(filename, index=False)
+    else:
+        print("Instruction file already exists at: %s" % (filename))
+
     print("Instruction file at: %s" % (filename))
 
     return filename