diff options
Diffstat (limited to 'tests')
| -rw-r--r-- | tests/checks.sh | 12 | ||||
| -rw-r--r-- | tests/config.sh | 14 | ||||
| -rw-r--r-- | tests/control.py | 28 | ||||
| -rwxr-xr-x[l---------] | tests/roof-vma.sh | 49 | ||||
| -rw-r--r-- | tests/roof.json | 19 | ||||
| -rwxr-xr-x | tests/roof.sh | 22 | ||||
| -rw-r--r-- | tests/roof/config.py | 2 | ||||
| -rw-r--r-- | tests/roof/graph.py | 10 | ||||
| -rwxr-xr-x | tests/vma-analyze.sh | 4 |
9 files changed, 130 insertions, 30 deletions
diff --git a/tests/checks.sh b/tests/checks.sh new file mode 100644 index 0000000..536bb42 --- /dev/null +++ b/tests/checks.sh @@ -0,0 +1,12 @@ +sudo tcpdump -i ens4d1 -e -c 1000 | cut -d ' ' -f 2- | sort | uniq +#nc -l -u -p 52080 + +#vma_stats $(ps xa | grep python3 | grep roof | awk '{print $1}') -c 1 -v 3 +# PSR - CPU core running process/thread +#ps -aeLF + + + +#echo 0 > /sys/kernel/mm/transparent_hugepage/khugepaged/defrag +#echo never > /sys/kernel/mm/transparent_hugepage/defrag +#echo never > /sys/kernel/mm/transparent_hugepage/enabled diff --git a/tests/config.sh b/tests/config.sh index 7b9d7d8..e0ee335 100644 --- a/tests/config.sh +++ b/tests/config.sh @@ -5,17 +5,13 @@ arch="" [ $el7 -ne 0 ] && arch="64" [ -f /etc/gentoo-release ] && arch="64" +ands_path=/mnt/ands +vma_lib_path=/usr/local/lib/ +vma_lib_ext="" -ods_path=/mnt/ands/ods/bin-fedora/ -vma_path=/mnt/ands/ -vma_lib_path=/mnt/ands/lib64-fedora/ +[ $el7 -eq 1 ] && vma_lib_path="${ands_path/fedora/centos}" -[ $el7 -eq 1 ] && ods_path="${ods_path/fedora/centos}" -[ $el7 -eq 1 ] && vma_lib_path="${vma_lib_path/fedora/centos}" - -# Standard library -#vma_lib=${vma_lib_path}/libvma.so.8.6.10 -vma_lib=${vma_lib_path}/libvma.so.8.9.5 +vma_lib=${vma_lib_path}/libvma.so${vma_lib_ext} # With Mellanox OFED extensions (./configure --enable-socketxtreme) #vma_lib=${vma_lib_path}/mlx/libvma.so.8.6.10 diff --git a/tests/control.py b/tests/control.py new file mode 100644 index 0000000..0d27f14 --- /dev/null +++ b/tests/control.py @@ -0,0 +1,28 @@ +import socket +import time + +#Communication with the boards (for configuration) happens via UDP messages to 192.168.100.xxx at port 51966. IP range is 192.168.100.100 to 192.168.100.119. +#If you need to toggle the streaming on/off, send a udp broadcast to 192.168.100.255 at port 51966 with udp payload x011100 (disable) or x011101 (enable). +#To reset the internal packet counter to 1, send x011500 +#To change packet size, send x0112XXXX with XXXX being the payload size. Received packets will always be 8 byte larger than this value because of the counter in the beginning of the payload. You should change the size only when streaming is turned off. +#Currently, all 20 devices are streaming data packets of 800 byte size at a rate ~500 MBit/s per device. They send to ports 52067 through 52086. +#I'm pretty sure it was configured correctly when I left. Just in case, you can use command hex 0106XXXXXXXXXXX with the mellanox MAC in hex as XX to set the target Mac address for the devices. Broadcast it to port 51966 at 192.168.100.255. + + +server = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) +#server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) +server.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) +#message = bytes.fromhex("011101") +#server.sendto(message, ("192.168.100.255", 51966)) + +message = bytes.fromhex("011100") +server.sendto(message, ("192.168.100.108", 51966)) +#server.sendto(message, ("192.168.100.119", 51966)) +server.sendto(message, ("192.168.100.106", 51966)) + + + +#sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) +#sock.bind(("0.0.0.0", 52068)) +#data, addr = sock.recvfrom(4096) +#print ("received message from %s of %i bytes " % (addr, len(data))) diff --git a/tests/roof-vma.sh b/tests/roof-vma.sh index 2faff84..3e97738 120000..100755 --- a/tests/roof-vma.sh +++ b/tests/roof-vma.sh @@ -1 +1,48 @@ -roof.sh
\ No newline at end of file +#! /bin/bash + +. config.sh + +function pyroof { +# numactl --cpunodebind=1 + LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/lib$arch" GI_TYPELIB_PATH="/usr/local/lib$arch/girepository-1.0/" \ + python3 roof.py "$@" +# numactl --cpunodebind=1 python3 roof.py "$@" +} + +if [[ "$0" =~ roof-vma ]]; then + function roof { + bufs=800000 + bufs=$((bufs * 4)) + + ulimit -l unlimited + echo 1000000000 > /proc/sys/kernel/shmmax # 18446744073692774399 + echo 8000 > /proc/sys/vm/nr_hugepages # 0 + + + tuned-adm profile latency-performance + cpupower frequency-set --governor performance # powersave + echo 100000000 > /proc/sys/kernel/sched_min_granularity_ns # 3000000 + echo 50000000 > /proc/sys/kernel/sched_migration_cost_ns # 500000 + echo 0 > /proc/sys/kernel/numa_balancing # 1 + + echo 0 > /proc/sys/vm/swappiness + sysctl -w vm.swappiness=0 + sysctl -w vm.zone_reclaim_mode=0 + echo never > /sys/kernel/mm/transparent_hugepage/enabled + + #VMA_SPEC=latency VMA_INTERNAL_THREAD_AFFINITY=1 + VMA_THREAD_MODE=3 VMA_MTU=0 VMA_RX_POLL=10 VMA_SELECT_POLL=10 VMA_RING_ALLOCATION_LOGIC_RX=20 VMA_MEM_ALLOC_TYPE=2 VMA_RX_BUFS=$bufs VMA_CQ_AIM_INTERRUPTS_RATE_PER_SEC=1000 LD_PRELOAD=$vma_lib \ + pyroof "$@" + } +else + function roof { + pyroof "$@" + } +fi + +#cat roof.yaml | sed '/simulation/,$d' | yq . > roof.json +#cat roof.yaml | yq r - -j | jq '' | sed -r '/\[$/ {:a;N;s/\]/&/;Ta;s/\n +//g;s/,(.)/, \1/}' > roof.json +#This is real +#cat roof.yaml | python3 yaml2json.py | sed -r '/\[$/ {:a;N;s/\]/&/;Ta;s/\n +//g;s/,(.)/, \1/}' > roof.json + +roof "$@" diff --git a/tests/roof.json b/tests/roof.json index 1848bc6..07c86f6 100644 --- a/tests/roof.json +++ b/tests/roof.json @@ -1,15 +1,15 @@ { "hardware": { "planes": 2, - "modules": 16, + "modules": 20, "bit_depth": 16, "channels_per_module": 16, "samples_per_rotation": 500 }, "geometry": { - "detector_diameter": [216, 216], - "source_diameter": [360, 365], - "source_angle": [270, 275], + "detector_diameter": 216, + "source_diameter": 360, + "source_angle": 270, "source_angle_offset": 3.2, "delta_x": 500, "delta_z": 1200 @@ -17,16 +17,19 @@ "network": { "protocol": "udp", "port": 52067, - "streams": 16, + "streams": 20, "header_size": 8, "payload_size": 800 }, "performance": { - "buffer_size": 10, - "packets_at_once": 100 + "buffer_size": 10000, + "packets_at_once": 500, + "latency_buffers": 1000, + "drop_buffers": 5000, + "sockets_per_thread": 1 }, "data": { - "base_path": "/home/csa/roof2_data/test_data" + "base_path": "/home/ufxray/roof/test_data" }, "simulation": { "first_file_number": 1, diff --git a/tests/roof.sh b/tests/roof.sh index a8fa89e..3e97738 100755 --- a/tests/roof.sh +++ b/tests/roof.sh @@ -3,8 +3,10 @@ . config.sh function pyroof { +# numactl --cpunodebind=1 LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/lib$arch" GI_TYPELIB_PATH="/usr/local/lib$arch/girepository-1.0/" \ - python3 roof.py "$@" + python3 roof.py "$@" +# numactl --cpunodebind=1 python3 roof.py "$@" } if [[ "$0" =~ roof-vma ]]; then @@ -16,7 +18,20 @@ if [[ "$0" =~ roof-vma ]]; then echo 1000000000 > /proc/sys/kernel/shmmax # 18446744073692774399 echo 8000 > /proc/sys/vm/nr_hugepages # 0 - VMA_THREAD_MODE=3 VMA_MTU=0 VMA_RX_POLL=0 VMA_SELECT_POLL=0 VMA_RING_ALLOCATION_LOGIC_RX=20 VMA_RX_BUFS=$bufs LD_PRELOAD=$vma_lib \ + + tuned-adm profile latency-performance + cpupower frequency-set --governor performance # powersave + echo 100000000 > /proc/sys/kernel/sched_min_granularity_ns # 3000000 + echo 50000000 > /proc/sys/kernel/sched_migration_cost_ns # 500000 + echo 0 > /proc/sys/kernel/numa_balancing # 1 + + echo 0 > /proc/sys/vm/swappiness + sysctl -w vm.swappiness=0 + sysctl -w vm.zone_reclaim_mode=0 + echo never > /sys/kernel/mm/transparent_hugepage/enabled + + #VMA_SPEC=latency VMA_INTERNAL_THREAD_AFFINITY=1 + VMA_THREAD_MODE=3 VMA_MTU=0 VMA_RX_POLL=10 VMA_SELECT_POLL=10 VMA_RING_ALLOCATION_LOGIC_RX=20 VMA_MEM_ALLOC_TYPE=2 VMA_RX_BUFS=$bufs VMA_CQ_AIM_INTERRUPTS_RATE_PER_SEC=1000 LD_PRELOAD=$vma_lib \ pyroof "$@" } else @@ -27,6 +42,7 @@ fi #cat roof.yaml | sed '/simulation/,$d' | yq . > roof.json #cat roof.yaml | yq r - -j | jq '' | sed -r '/\[$/ {:a;N;s/\]/&/;Ta;s/\n +//g;s/,(.)/, \1/}' > roof.json -cat roof.yaml | python3 yaml2json.py | sed -r '/\[$/ {:a;N;s/\]/&/;Ta;s/\n +//g;s/,(.)/, \1/}' > roof.json +#This is real +#cat roof.yaml | python3 yaml2json.py | sed -r '/\[$/ {:a;N;s/\]/&/;Ta;s/\n +//g;s/,(.)/, \1/}' > roof.json roof "$@" diff --git a/tests/roof/config.py b/tests/roof/config.py index 5ea8d41..ace895a 100644 --- a/tests/roof/config.py +++ b/tests/roof/config.py @@ -22,6 +22,8 @@ class RoofConfig: self.fan_bins = self.modules * self.get_opt('hardware', 'channels_per_module', 16) self.fan_projections = self.get_opt('hardware', 'samples_per_rotation', (self.sample_rate / self.imaging_rate) if (self.imaging_rate and self.sample_rate) else 1000) + self.sockets_per_thread = self.get_opt('performance', 'sockets_per_stream', 1) + if self.args.number is None: self.args.number = 0 if self.args.benchmark else self.planes # Consistency and default mode diff --git a/tests/roof/graph.py b/tests/roof/graph.py index 62aa44e..0f516bc 100644 --- a/tests/roof/graph.py +++ b/tests/roof/graph.py @@ -1,3 +1,4 @@ +import math import re import gi @@ -57,7 +58,7 @@ class RoofGraph(RoofConfig): params = { 'path': path, 'first': first, 'step': step } if self.args.number: params['number'] = self.args.number - + print ("Reading {} data from {}".format(self.args.read,path)) # FIXME: handle raw data parameters return self.get_task('read', **params) @@ -72,12 +73,7 @@ class RoofGraph(RoofConfig): # Reconstruction from network or simulated data (also generation of flat/dark-fields) build_type = "raw" if self.args.noroof else "sino" if self.check_writer_type_is_raw() else "ufo" - build = self.get_roof_task('roof-build', simulate = self.args.simulate, number = self.args.number, build = build_type) - for id in range(self.streams): - read = self.get_roof_task('roof-read', id = id, simulate = self.args.simulate, path = path, first_file_number = first) - self.graph.connect_nodes(read, build) - build.bind_property('stop', read, 'stop', GObject.BindingFlags.DEFAULT) - + build = self.get_roof_task('roof-build', simulate = self.args.simulate, path = path, first_file_number = first, number = self.args.number, build = build_type) return build def get_writer(self): diff --git a/tests/vma-analyze.sh b/tests/vma-analyze.sh index 7000922..77c4eb1 100755 --- a/tests/vma-analyze.sh +++ b/tests/vma-analyze.sh @@ -7,9 +7,9 @@ sleep=1 path=/mnt/ands/bin/vma_stats_mlx #-z seems ignored #$path -p $(pidof onlineDetectorSimulatorServer) -c 1 -z &> /dev/null -stats1=($($path -p $(pidof python) -c 1 | grep Rx | awk '{ print $3, $4 }')) +stats1=($($path -p $(ps xa | grep python3 | grep roof | awk '{print $1}') -c 1 | grep Rx | awk '{ print $3, $4 }')) sleep $sleep -stats2=($($path -p $(pidof python) -c 1 | grep Rx | awk '{ print $3, $4 }')) +stats2=($($path -p $(ps xa | grep pyth | grep roof | awk '{print $1}') -c 1 | grep Rx | awk '{ print $3, $4 }')) pksum=0 bwsum=0 |