Skip to content
Snippets Groups Projects
Commit ee73cc25 authored by Cookiebowser's avatar Cookiebowser
Browse files

added ProB-MC results and updated benchmark-processing skript

parent b0fd0744
Branches rust-support
No related tags found
1 merge request!28Rust support
Show whitespace changes
Inline Side-by-side
benchmarks/average_benchmark_results.py
+ 12
− 4
View file @ ee73cc25
......@@ -6,9 +6,13 @@ file = sys.argv[1]
print('opening file: ', file)
full_print = True
if len(sys.argv) > 2 and sys.argv[2].upper() == 'SHORT':
print('argv2 provided')
mc = True
for arg in sys.argv[2:]:
if arg.upper() == 'SHORT':
full_print = False
elif arg.upper() == 'NO_MC':
mc = False
# {machine: { (mode: str, threads: int, caching: bool): {times: [], memory: []} }}
data = {}
......@@ -19,7 +23,11 @@ with open(file, 'r') as lines:
stripped_line = line.strip()
if stripped_line == '' or stripped_line.startswith('-'):
continue
machine, mode, threads, caching, time, memory = ('', '', 1, 'False', 0, '')
if mc:
machine, mode, threads, caching, time, memory = stripped_line.split(' ')
else:
machine, time, memory = stripped_line.split(' ')
if machine not in data:
data[machine] = {}
params = (mode, int(threads), caching.upper() == 'TRUE')
......
benchmarks/model_checking/README.md
+ 58
− 58
View file @ ee73cc25
......@@ -61,34 +61,34 @@ Executing benchmarks is done by executing ``make`` in each of the directories.
#### System B
| Machines | | ProB OP | ProB ST | Java | C++ -O1 | C++ -O2 | Rust |
|-------------------|----------|---------|---------|------------------------|-----------------------|-----------------------|-----------------------|
| Lift_MC_Large | Runtime | | | 2.23 / 2.7 | 0.80 / 0.97 | 0.74 / 00.96 | 00.42 / 00.50 |
| | Speed-up | | | | | | |
| | Memory | | | 700.824 / 721.548 | 250.488 / 250.524 | 250.388 / 250.405 | 54.734 / 54.742 |
| Cruise Controller | Runtime | | | 1.73 / 2.28 | 0.14 / 0.14 | 0.11 / 00.12 | 00.07 / 00.18 |
| | Speed-up | | | | | | |
| | Memory | | | 159.292 / 161.860 | 6.296 / 6.580 | 6.310 / 6.489 | 5011 / 5226 |
| Landing Gear | Runtime | | | 11.16 / 13.52 | 10.95 / 10.58 | 8.79 / 9.59 | 09.84 / 19.23 |
| | Speed-up | | | | | | |
| | Memory | | | 1.063.784 / 1.352.408 | 329.680 / 333.076 | 329.672 / 332.976 | 734.749 / 854.914 |
| CAN BUS | Runtime | | | 3.79 / 4.47 | 1.99 / 2.1 | 1.71 / 2.02 | 02.3 / 02.45 |
| | Speed-up | | | | | | |
| | Memory | | | 542.680 / 1.080.708 | 316.888 / 326.688 | 316.873 / 326.791 | 1.012.179 / 1.049.515 |
| Train1_Lukas | Runtime | | | 13.0 / 12.25 | 5.29 / 2.39 | 4.93 / 2.31 | 07.08 / 07.14 |
| | Speed-up | | | | | | |
| | Memory | | | 701.600 / 1.044.600 | 59.408 / 62.912 | 59.299 / 62.798 | 187.201 / 255.626 |
| Train_1_beebook | Runtime | | | 626.39 / 545.66 | 338.03 / 140.84 | 311.83 / 135.69 | 445.79 / 416.89 |
| | Speed-up | | | | | | |
| | Memory | | | 1.451.340 / 3.110.416 | 1.559.736 / 1.667.508 | 1.559.626 / 1.667.335 | 4.735.570 / 6.047.158 |
| nota | Runtime | | | 10.75 / 9.58 | 26.55 / 23.1 | 22.43 / 21.4 | 20.12 / 20.77 |
| | Speed-up | | | | | | |
| | Memory | | | 1.201.656 / 1.512.316 | 280.532 / 413.696 | 280.518 / 413.597 | 1.321.891 / 1.269.008 |
| sort_1000 | Runtime | | | 988.25 / 85.87 | 1650.41 / 1317.36 | 1424.95 / 1251.89 | 516 / 95.12 |
| | Speed-up | | | | | | |
| | Memory | | | 1.202.264 / 1.361.680 | 338.680 / 342.100 | 338.626 / 342.083 | 269.996 / 269.939 |
| N Queens (N=4) | Runtime | | | 34.41 / 35.73 | 28.81 / 32.6 | 27.79 / 27.16 | 22.34 / 22.74 |
| | Speed-up | | | | | | |
| | Memory | | | 511.568 / 503.032 | 61.684 / 61.980 | 61.762 / 61.814 | 345.376 / 345.386 |
|--------------------|----------|-----------|-----------|-----------------------|-----------------------|-----------------------|-----------------------|
| Lift_MC_Large | Runtime | 89.59 | 84.3 | 2.23 / 2.7 | 0.80 / 0.97 | 0.74 / 00.96 | 00.42 / 00.50 |
| | Speed-up | 1 | 1.06 | 40.17 / 33.18 | 111.99 / 92.36 | 121.07 / 93.32 | 213.31 / 179.18 |
| | Memory | 1.139.020 | 1.139.546 | 700.824 / 721.548 | 250.488 / 250.524 | 250.388 / 250.405 | 54.734 / 54.742 |
| Cruise Controller | Runtime | 2.7 | 3.56 | 1.73 / 2.28 | 0.14 / 0.14 | 0.11 / 00.12 | 00.07 / 00.18 |
| | Speed-up | 1 | 0.76 | 1.56 / 1.18 | 19.29 / 19.29 | 24.55 / 22.5 | 38.57 / 15 |
| | Memory | 186.172 | 186.408 | 159.292 / 161.860 | 6.296 / 6.580 | 6.310 / 6.489 | 5011 / 5226 |
| Landing Gear | Runtime | 45.35 | 185.56 | 11.16 / 13.52 | 10.95 / 10.58 | 8.79 / 9.59 | 09.84 / 19.23 |
| | Speed-up | 1 | 0.24 | 4.06 / 3.35 | 4.14 / 4.29 | 5.16 / 4.73 | 4.61 / 2.36 |
| | Memory | 487.388 | 481.932 | 1.063.784 / 1.352.408 | 329.680 / 333.076 | 329.672 / 332.976 | 734.749 / 854.914 |
| CAN BUS | Runtime | 27.78 | 52.99 | 3.79 / 4.47 | 1.99 / 2.1 | 1.71 / 2.02 | 02.3 / 02.45 |
| | Speed-up | 1 | 0.52 | 7.33 / 6.21 | 13.96 / 13.23 | 16.25 / 13.75 | 12.08 / 11.34 |
| | Memory | 366.300 | 365.156 | 542.680 / 1.080.708 | 316.888 / 326.688 | 316.873 / 326.791 | 1.012.179 / 1.049.515 |
| Train1_Lukas | Runtime | 21.59 | 54.19 | 13.0 / 12.25 | 5.29 / 2.39 | 4.93 / 2.31 | 07.08 / 07.14 |
| | Speed-up | 1 | 0.4 | 1.66 / 1.76 | 4.08 / 9.03 | 4.38 / 9.35 | 3.05 / 3.02 |
| | Memory | 289.692 | 217.156 | 701.600 / 1.044.600 | 59.408 / 62.912 | 59.299 / 62.798 | 187.201 / 255.626 |
| Train_1_beebook_v2 | Runtime | 570.51 | 1642.6 | 626.39 / 545.66 | 338.03 / 140.84 | 311.83 / 135.69 | 445.79 / 416.89 |
| | Speed-up | 1 | 0.35 | 0.91 / 1.05 | 1.69 / 4.05 | 1.83 / 4.2 | 1.28 / 1.37 |
| | Memory | 2.998.192 | 1.310.184 | 1.451.340 / 3.110.416 | 1.559.736 / 1.667.508 | 1.559.626 / 1.667.335 | 4.735.570 / 6.047.158 |
| nota | Runtime | 33.08 | 166.53 | 10.75 / 9.58 | 26.55 / 23.1 | 22.43 / 21.4 | 20.12 / 20.77 |
| | Speed-up | 1 | 0.2 | 3.08 / 3.45 | 1.25 / 1.43 | 1.47 / 1.55 | 1.64 / 1.59 |
| | Memory | 950.228 | 948.460 | 1.201.656 / 1.512.316 | 280.532 / 413.696 | 280.518 / 413.597 | 1.321.891 / 1.269.008 |
| sort_1000 | Runtime | 242.17 | 301.23 | 988.25 / 85.87 | 1650.41 / 1317.36 | 1424.95 / 1251.89 | 516 / 95.12 |
| | Speed-up | 1 | 0.8 | 0.25 / 2.82 | 0.15 / 0.18 | 0.17 / 0.19 | 0.47 / 2.55 |
| | Memory | 828.468 | 602.360 | 1.202.264 / 1.361.680 | 338.680 / 342.100 | 338.626 / 342.083 | 269.996 / 269.939 |
| N Queens (N=4) | Runtime | 2.15 | 2.18 | 34.41 / 35.73 | 28.81 / 32.6 | 27.79 / 27.16 | 22.34 / 22.74 |
| | Speed-up | 1 | 0.99 | 0.06 / 0.06 | 0.07 / 0.07 | 0.08 / 0.08 | 0.1 / 0.09 |
| | Memory | 178.152 | 177.208 | 511.568 / 503.032 | 61.684 / 61.980 | 61.762 / 61.814 | 345.376 / 345.386 |
### Multi-threaded (6 Threads)
......@@ -140,35 +140,35 @@ Current implementation requires an additional coordinator thread.
#### System B
| Machines | | ProB OP | ProB ST | Java | C++ -O1 | C++ -O2 | Rust |
|-------------------|----------|---------|---------|-----------------------|-----------------------|-----------------------|-----------------------|
| Lift_MC_Large | Runtime | | | 67.14 / 67.37 | 63.33 / 64.33 | 64.37 / 61.59 | 09.82 / 09.96 |
| | Speed-up | | | | | | |
| | Memory | | | 548.128 / 522.388 | 250.840 / 250.926 | 250.822 / 250.832 | 38.637 / 38.590 |
| Cruise Controller | Runtime | | | 1.76 / 1.78 | 0.06 / 0.06 | 0.05 / 00.06 | 00.04 / 00.06 |
| | Speed-up | | | | | | |
| | Memory | | | 171.916 / 180.132 | 6.544 / 7.112 | 6.697 / 7.018 | 6.978 / 6.959 |
| Landing Gear | Runtime | | | 6.42 / 6.69 | 7.28 / 7.59 | 7.03 / 6.76 | 03.41 / 04.56 |
| | Speed-up | | | | | | |
| | Memory | | | 1.405.280 / 2.043.336 | 340.408 / 345.720 | 341.353 / 345.930 | 727.461 / 847.765 |
| CAN BUS | Runtime | | | 3.52 / 4.29 | 3.92 / 4.51 | 4.12 / 4.35 | 01.64 / 01.64 |
| | Speed-up | | | | | | |
| | Memory | | | 550.004 / 1.159.472 | 317.140 / 326.652 | 317.437 / 326.536 | 1.047.918 / 1.050.520 |
| Train1_Lukas | Runtime | | | 5.96 / 5.44 | 4.46 / 2.02 | 4.36 / 1.91 | 01.61 / 01.46 |
| | Speed-up | | | | | |
| | Memory | | | 1.468.360 / 1.772.524 | 60.412 / 65.012 | 60.320 / 64.956 | 189.500 / 250.548 |
| Train_1_beebook | Runtime | | | 174.56 / 147.29 | 257.59 / 115.41 | 250.92 / 111.71 | 89.65 / 76.61 |
| | Speed-up | | | | | | |
| | Memory | | | 2.555.344 / 3.094.376 | 1.573.868 / 1.812.972 | 1.572.735 / 1.814.287 | 4.998.943 / 5.484.696 |
| nota | Runtime | | | 5.12 / 4.74 | 25.91 / 25.63 | 26.12 / 21.58 | 05.02 / 05.24 |
| | Speed-up | | | | | | |
| | Memory | | | 2.185.468 / 2.464.928 | 300.208 / 438.692 | 299.167 / 433.206 | 1.390.200 / 1.369.317 |
| sort_1000 | Runtime | | | 208.82 / 37.32 | 327.81 / 279.46 | 336.3 / 273.14 | 110.91 / 27.47 |
| | Speed-up | | | | | | |
| | Memory | | | 2.359.820 / 2.420.428 | 323.472 / 341.740 | 323.596 / 341.784 | 243.784 / 243.330 |
| N Queens (N=4) | Runtime | | | 32.87 / 32.2 | 30.02 / 28.66 | 30.88 / 25.98 | 21.34 / 22.46 |
| | Speed-up | | | | | | |
| | Memory | | | 508.736 / 499.924 | 62696 / 62.832 | 62.630 / 62.742 | 345.888 / 345.855 |
| Machines | | Java | C++ -O1 | C++ -O2 | Rust |
|-------------------|----------|-----------------------|-----------------------|-----------------------|-----------------------|
| Lift_MC_Large | Runtime | 67.14 / 67.37 | 63.33 / 64.33 | 64.37 / 61.59 | 09.82 / 09.96 |
| | Speed-up | | | | |
| | Memory | 548.128 / 522.388 | 250.840 / 250.926 | 250.822 / 250.832 | 38.637 / 38.590 |
| Cruise Controller | Runtime | 1.76 / 1.78 | 0.06 / 0.06 | 0.05 / 00.06 | 00.04 / 00.06 |
| | Speed-up | | | | |
| | Memory | 171.916 / 180.132 | 6.544 / 7.112 | 6.697 / 7.018 | 6.978 / 6.959 |
| Landing Gear | Runtime | 6.42 / 6.69 | 7.28 / 7.59 | 7.03 / 6.76 | 03.41 / 04.56 |
| | Speed-up | | | | |
| | Memory | 1.405.280 / 2.043.336 | 340.408 / 345.720 | 341.353 / 345.930 | 727.461 / 847.765 |
| CAN BUS | Runtime | 3.52 / 4.29 | 3.92 / 4.51 | 4.12 / 4.35 | 01.64 / 01.64 |
| | Speed-up | | | | |
| | Memory | 550.004 / 1.159.472 | 317.140 / 326.652 | 317.437 / 326.536 | 1.047.918 / 1.050.520 |
| Train1_Lukas | Runtime | 5.96 / 5.44 | 4.46 / 2.02 | 4.36 / 1.91 | 01.61 / 01.46 |
| | Speed-up | | | |
| | Memory | 1.468.360 / 1.772.524 | 60.412 / 65.012 | 60.320 / 64.956 | 189.500 / 250.548 |
| Train_1_beebook | Runtime | 174.56 / 147.29 | 257.59 / 115.41 | 250.92 / 111.71 | 89.65 / 76.61 |
| | Speed-up | | | | |
| | Memory | 2.555.344 / 3.094.376 | 1.573.868 / 1.812.972 | 1.572.735 / 1.814.287 | 4.998.943 / 5.484.696 |
| nota | Runtime | 5.12 / 4.74 | 25.91 / 25.63 | 26.12 / 21.58 | 05.02 / 05.24 |
| | Speed-up | | | | |
| | Memory | 2.185.468 / 2.464.928 | 300.208 / 438.692 | 299.167 / 433.206 | 1.390.200 / 1.369.317 |
| sort_1000 | Runtime | 208.82 / 37.32 | 327.81 / 279.46 | 336.3 / 273.14 | 110.91 / 27.47 |
| | Speed-up | | | | |
| | Memory | 2.359.820 / 2.420.428 | 323.472 / 341.740 | 323.596 / 341.784 | 243.784 / 243.330 |
| N Queens (N=4) | Runtime | 32.87 / 32.2 | 30.02 / 28.66 | 30.88 / 25.98 | 21.34 / 22.46 |
| | Speed-up | | | | |
| | Memory | 508.736 / 499.924 | 62696 / 62.832 | 62.630 / 62.742 | 345.888 / 345.855 |
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment