-
Notifications
You must be signed in to change notification settings - Fork 1
/
run.sh
executable file
·40 lines (33 loc) · 980 Bytes
/
run.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
#!/bin/zsh
if [ "$#" -ne 1 ]; then
echo "Illegal number of parameters"
exit -1
fi
file="datasets/alexa-top1m-2021-01-04_0900_UTC.csv"
num_files=$1
((num_processes=num_files-1))
total_lines=$(wc -l <$file)
((lines_per_file = (total_lines + num_files - 1) / num_files))
echo "Lines per file = ${lines_per_file}"
echo "Number of processes = ${num_processes}"
if [[ "$(uname)" == "Darwin" ]]; then
# On macos the split command is prefixed with a 'g'
gsplit --lines ${lines_per_file} -a4 -d ${file} "datasets/domains."
else
split --lines ${lines_per_file} -a4 -d ${file} "datasets/domains."
fi
pids=()
for i in {0000..$num_processes}; do
current_input="datasets/domains.${i}"
current_output="output/results_${i}.json"
probing --input ${current_input} --output ${current_output} &
pids+=($!)
printf "$! "
done;
echo ""
# wait for all pids
for pid in ${pids[*]}; do
wait $pid
done
cat "output/results_*" > "output/result.json"
rm -rf "output/results_*"