ntb_test.sh 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590
  1. #!/bin/bash
  2. # Copyright (c) 2016 Microsemi. All Rights Reserved.
  3. #
  4. # This program is free software; you can redistribute it and/or
  5. # modify it under the terms of the GNU General Public License as
  6. # published by the Free Software Foundation; either version 2 of
  7. # the License, or (at your option) any later version.
  8. #
  9. # This program is distributed in the hope that it would be useful,
  10. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12. # GNU General Public License for more details.
  13. #
  14. # Author: Logan Gunthorpe <logang@deltatee.com>
  15. REMOTE_HOST=
  16. LIST_DEVS=FALSE
  17. DEBUGFS=${DEBUGFS-/sys/kernel/debug}
  18. PERF_RUN_ORDER=32
  19. MAX_MW_SIZE=0
  20. RUN_DMA_TESTS=
  21. DONT_CLEANUP=
  22. MW_SIZE=65536
  23. function show_help()
  24. {
  25. echo "Usage: $0 [OPTIONS] LOCAL_DEV REMOTE_DEV"
  26. echo "Run tests on a pair of NTB endpoints."
  27. echo
  28. echo "If the NTB device loops back to the same host then,"
  29. echo "just specifying the two PCI ids on the command line is"
  30. echo "sufficient. Otherwise, if the NTB link spans two hosts"
  31. echo "use the -r option to specify the hostname for the remote"
  32. echo "device. SSH will then be used to test the remote side."
  33. echo "An SSH key between the root users of the host would then"
  34. echo "be highly recommended."
  35. echo
  36. echo "Options:"
  37. echo " -C don't cleanup ntb modules on exit"
  38. echo " -h show this help message"
  39. echo " -l list available local and remote PCI ids"
  40. echo " -r REMOTE_HOST specify the remote's hostname to connect"
  41. echo " to for the test (using ssh)"
  42. echo " -m MW_SIZE memory window size for ntb_tool"
  43. echo " (default: $MW_SIZE)"
  44. echo " -d run dma tests for ntb_perf"
  45. echo " -p ORDER total data order for ntb_perf"
  46. echo " (default: $PERF_RUN_ORDER)"
  47. echo " -w MAX_MW_SIZE maxmium memory window size for ntb_perf"
  48. echo
  49. }
  50. function parse_args()
  51. {
  52. OPTIND=0
  53. while getopts "b:Cdhlm:r:p:w:" opt; do
  54. case "$opt" in
  55. C) DONT_CLEANUP=1 ;;
  56. d) RUN_DMA_TESTS=1 ;;
  57. h) show_help; exit 0 ;;
  58. l) LIST_DEVS=TRUE ;;
  59. m) MW_SIZE=${OPTARG} ;;
  60. r) REMOTE_HOST=${OPTARG} ;;
  61. p) PERF_RUN_ORDER=${OPTARG} ;;
  62. w) MAX_MW_SIZE=${OPTARG} ;;
  63. \?)
  64. echo "Invalid option: -$OPTARG" >&2
  65. exit 1
  66. ;;
  67. esac
  68. done
  69. }
  70. parse_args "$@"
  71. shift $((OPTIND-1))
  72. LOCAL_DEV=$1
  73. shift
  74. parse_args "$@"
  75. shift $((OPTIND-1))
  76. REMOTE_DEV=$1
  77. shift
  78. parse_args "$@"
  79. set -e
  80. function _modprobe()
  81. {
  82. modprobe "$@"
  83. if [[ "$REMOTE_HOST" != "" ]]; then
  84. ssh "$REMOTE_HOST" modprobe "$@"
  85. fi
  86. }
  87. function split_remote()
  88. {
  89. VPATH=$1
  90. REMOTE=
  91. if [[ "$VPATH" == *":/"* ]]; then
  92. REMOTE=${VPATH%%:*}
  93. VPATH=${VPATH#*:}
  94. fi
  95. }
  96. function read_file()
  97. {
  98. split_remote $1
  99. if [[ "$REMOTE" != "" ]]; then
  100. ssh "$REMOTE" cat "$VPATH"
  101. else
  102. cat "$VPATH"
  103. fi
  104. }
  105. function write_file()
  106. {
  107. split_remote $2
  108. VALUE=$1
  109. if [[ "$REMOTE" != "" ]]; then
  110. ssh "$REMOTE" "echo \"$VALUE\" > \"$VPATH\""
  111. else
  112. echo "$VALUE" > "$VPATH"
  113. fi
  114. }
  115. function check_file()
  116. {
  117. split_remote $1
  118. if [[ "$REMOTE" != "" ]]; then
  119. ssh "$REMOTE" "[[ -e ${VPATH} ]]"
  120. else
  121. [[ -e ${VPATH} ]]
  122. fi
  123. }
  124. function subdirname()
  125. {
  126. echo $(basename $(dirname $1)) 2> /dev/null
  127. }
  128. function find_pidx()
  129. {
  130. PORT=$1
  131. PPATH=$2
  132. for ((i = 0; i < 64; i++)); do
  133. PEER_DIR="$PPATH/peer$i"
  134. check_file ${PEER_DIR} || break
  135. PEER_PORT=$(read_file "${PEER_DIR}/port")
  136. if [[ ${PORT} -eq $PEER_PORT ]]; then
  137. echo $i
  138. return 0
  139. fi
  140. done
  141. return 1
  142. }
  143. function port_test()
  144. {
  145. LOC=$1
  146. REM=$2
  147. echo "Running port tests on: $(basename $LOC) / $(basename $REM)"
  148. LOCAL_PORT=$(read_file "$LOC/port")
  149. REMOTE_PORT=$(read_file "$REM/port")
  150. LOCAL_PIDX=$(find_pidx ${REMOTE_PORT} "$LOC")
  151. REMOTE_PIDX=$(find_pidx ${LOCAL_PORT} "$REM")
  152. echo "Local port ${LOCAL_PORT} with index ${REMOTE_PIDX} on remote host"
  153. echo "Peer port ${REMOTE_PORT} with index ${LOCAL_PIDX} on local host"
  154. echo " Passed"
  155. }
  156. function link_test()
  157. {
  158. LOC=$1
  159. REM=$2
  160. EXP=0
  161. echo "Running link tests on: $(subdirname $LOC) / $(subdirname $REM)"
  162. if ! write_file "N" "$LOC/../link" 2> /dev/null; then
  163. echo " Unsupported"
  164. return
  165. fi
  166. write_file "N" "$LOC/link_event"
  167. if [[ $(read_file "$REM/link") != "N" ]]; then
  168. echo "Expected link to be down in $REM/link" >&2
  169. exit -1
  170. fi
  171. write_file "Y" "$LOC/../link"
  172. echo " Passed"
  173. }
  174. function doorbell_test()
  175. {
  176. LOC=$1
  177. REM=$2
  178. EXP=0
  179. echo "Running db tests on: $(basename $LOC) / $(basename $REM)"
  180. DB_VALID_MASK=$(read_file "$LOC/db_valid_mask")
  181. write_file "c $DB_VALID_MASK" "$REM/db"
  182. for ((i = 0; i < 64; i++)); do
  183. DB=$(read_file "$REM/db")
  184. if [[ "$DB" -ne "$EXP" ]]; then
  185. echo "Doorbell doesn't match expected value $EXP " \
  186. "in $REM/db" >&2
  187. exit -1
  188. fi
  189. let "MASK = (1 << $i) & $DB_VALID_MASK" || true
  190. let "EXP = $EXP | $MASK" || true
  191. write_file "s $MASK" "$LOC/peer_db"
  192. done
  193. write_file "c $DB_VALID_MASK" "$REM/db_mask"
  194. write_file $DB_VALID_MASK "$REM/db_event"
  195. write_file "s $DB_VALID_MASK" "$REM/db_mask"
  196. write_file "c $DB_VALID_MASK" "$REM/db"
  197. echo " Passed"
  198. }
  199. function get_files_count()
  200. {
  201. NAME=$1
  202. LOC=$2
  203. split_remote $LOC
  204. if [[ "$REMOTE" == "" ]]; then
  205. echo $(ls -1 "$VPATH"/${NAME}* 2>/dev/null | wc -l)
  206. else
  207. echo $(ssh "$REMOTE" "ls -1 \"$VPATH\"/${NAME}* | \
  208. wc -l" 2> /dev/null)
  209. fi
  210. }
  211. function scratchpad_test()
  212. {
  213. LOC=$1
  214. REM=$2
  215. echo "Running spad tests on: $(subdirname $LOC) / $(subdirname $REM)"
  216. CNT=$(get_files_count "spad" "$LOC")
  217. if [[ $CNT -eq 0 ]]; then
  218. echo " Unsupported"
  219. return
  220. fi
  221. for ((i = 0; i < $CNT; i++)); do
  222. VAL=$RANDOM
  223. write_file "$VAL" "$LOC/spad$i"
  224. RVAL=$(read_file "$REM/../spad$i")
  225. if [[ "$VAL" -ne "$RVAL" ]]; then
  226. echo "Scratchpad $i value $RVAL doesn't match $VAL" >&2
  227. exit -1
  228. fi
  229. done
  230. echo " Passed"
  231. }
  232. function message_test()
  233. {
  234. LOC=$1
  235. REM=$2
  236. echo "Running msg tests on: $(subdirname $LOC) / $(subdirname $REM)"
  237. CNT=$(get_files_count "msg" "$LOC")
  238. if [[ $CNT -eq 0 ]]; then
  239. echo " Unsupported"
  240. return
  241. fi
  242. MSG_OUTBITS_MASK=$(read_file "$LOC/../msg_inbits")
  243. MSG_INBITS_MASK=$(read_file "$REM/../msg_inbits")
  244. write_file "c $MSG_OUTBITS_MASK" "$LOC/../msg_sts"
  245. write_file "c $MSG_INBITS_MASK" "$REM/../msg_sts"
  246. for ((i = 0; i < $CNT; i++)); do
  247. VAL=$RANDOM
  248. write_file "$VAL" "$LOC/msg$i"
  249. RVAL=$(read_file "$REM/../msg$i")
  250. if [[ "$VAL" -ne "${RVAL%%<-*}" ]]; then
  251. echo "Message $i value $RVAL doesn't match $VAL" >&2
  252. exit -1
  253. fi
  254. done
  255. echo " Passed"
  256. }
  257. function get_number()
  258. {
  259. KEY=$1
  260. sed -n "s/^\(${KEY}\)[ \t]*\(0x[0-9a-fA-F]*\)\(\[p\]\)\?$/\2/p"
  261. }
  262. function mw_alloc()
  263. {
  264. IDX=$1
  265. LOC=$2
  266. REM=$3
  267. write_file $MW_SIZE "$LOC/mw_trans$IDX"
  268. INB_MW=$(read_file "$LOC/mw_trans$IDX")
  269. MW_ALIGNED_SIZE=$(echo "$INB_MW" | get_number "Window Size")
  270. MW_DMA_ADDR=$(echo "$INB_MW" | get_number "DMA Address")
  271. write_file "$MW_DMA_ADDR:$(($MW_ALIGNED_SIZE))" "$REM/peer_mw_trans$IDX"
  272. if [[ $MW_SIZE -ne $MW_ALIGNED_SIZE ]]; then
  273. echo "MW $IDX size aligned to $MW_ALIGNED_SIZE"
  274. fi
  275. }
  276. function write_mw()
  277. {
  278. split_remote $2
  279. if [[ "$REMOTE" != "" ]]; then
  280. ssh "$REMOTE" \
  281. dd if=/dev/urandom "of=$VPATH" 2> /dev/null || true
  282. else
  283. dd if=/dev/urandom "of=$VPATH" 2> /dev/null || true
  284. fi
  285. }
  286. function mw_check()
  287. {
  288. IDX=$1
  289. LOC=$2
  290. REM=$3
  291. write_mw "$LOC/mw$IDX"
  292. split_remote "$LOC/mw$IDX"
  293. if [[ "$REMOTE" == "" ]]; then
  294. A=$VPATH
  295. else
  296. A=/tmp/ntb_test.$$.A
  297. ssh "$REMOTE" cat "$VPATH" > "$A"
  298. fi
  299. split_remote "$REM/peer_mw$IDX"
  300. if [[ "$REMOTE" == "" ]]; then
  301. B=$VPATH
  302. else
  303. B=/tmp/ntb_test.$$.B
  304. ssh "$REMOTE" cat "$VPATH" > "$B"
  305. fi
  306. cmp -n $MW_ALIGNED_SIZE "$A" "$B"
  307. if [[ $? != 0 ]]; then
  308. echo "Memory window $MW did not match!" >&2
  309. fi
  310. if [[ "$A" == "/tmp/*" ]]; then
  311. rm "$A"
  312. fi
  313. if [[ "$B" == "/tmp/*" ]]; then
  314. rm "$B"
  315. fi
  316. }
  317. function mw_free()
  318. {
  319. IDX=$1
  320. LOC=$2
  321. REM=$3
  322. write_file "$MW_DMA_ADDR:0" "$REM/peer_mw_trans$IDX"
  323. write_file 0 "$LOC/mw_trans$IDX"
  324. }
  325. function mw_test()
  326. {
  327. LOC=$1
  328. REM=$2
  329. CNT=$(get_files_count "mw_trans" "$LOC")
  330. for ((i = 0; i < $CNT; i++)); do
  331. echo "Running mw$i tests on: $(subdirname $LOC) / " \
  332. "$(subdirname $REM)"
  333. mw_alloc $i $LOC $REM
  334. mw_check $i $LOC $REM
  335. mw_free $i $LOC $REM
  336. echo " Passed"
  337. done
  338. }
  339. function pingpong_test()
  340. {
  341. LOC=$1
  342. REM=$2
  343. echo "Running ping pong tests on: $(basename $LOC) / $(basename $REM)"
  344. LOC_START=$(read_file "$LOC/count")
  345. REM_START=$(read_file "$REM/count")
  346. sleep 7
  347. LOC_END=$(read_file "$LOC/count")
  348. REM_END=$(read_file "$REM/count")
  349. if [[ $LOC_START == $LOC_END ]] || [[ $REM_START == $REM_END ]]; then
  350. echo "Ping pong counter not incrementing!" >&2
  351. exit 1
  352. fi
  353. echo " Passed"
  354. }
  355. function perf_test()
  356. {
  357. USE_DMA=$1
  358. if [[ $USE_DMA == "1" ]]; then
  359. WITH="with"
  360. else
  361. WITH="without"
  362. fi
  363. _modprobe ntb_perf total_order=$PERF_RUN_ORDER \
  364. max_mw_size=$MAX_MW_SIZE use_dma=$USE_DMA
  365. echo "Running local perf test $WITH DMA"
  366. write_file "$LOCAL_PIDX" "$LOCAL_PERF/run"
  367. echo -n " "
  368. read_file "$LOCAL_PERF/run"
  369. echo " Passed"
  370. echo "Running remote perf test $WITH DMA"
  371. write_file "$REMOTE_PIDX" "$REMOTE_PERF/run"
  372. echo -n " "
  373. read_file "$REMOTE_PERF/run"
  374. echo " Passed"
  375. _modprobe -r ntb_perf
  376. }
  377. function ntb_tool_tests()
  378. {
  379. LOCAL_TOOL="$DEBUGFS/ntb_tool/$LOCAL_DEV"
  380. REMOTE_TOOL="$REMOTE_HOST:$DEBUGFS/ntb_tool/$REMOTE_DEV"
  381. echo "Starting ntb_tool tests..."
  382. _modprobe ntb_tool
  383. port_test "$LOCAL_TOOL" "$REMOTE_TOOL"
  384. LOCAL_PEER_TOOL="$LOCAL_TOOL/peer$LOCAL_PIDX"
  385. REMOTE_PEER_TOOL="$REMOTE_TOOL/peer$REMOTE_PIDX"
  386. link_test "$LOCAL_PEER_TOOL" "$REMOTE_PEER_TOOL"
  387. link_test "$REMOTE_PEER_TOOL" "$LOCAL_PEER_TOOL"
  388. #Ensure the link is up on both sides before continuing
  389. write_file "Y" "$LOCAL_PEER_TOOL/link_event"
  390. write_file "Y" "$REMOTE_PEER_TOOL/link_event"
  391. doorbell_test "$LOCAL_TOOL" "$REMOTE_TOOL"
  392. doorbell_test "$REMOTE_TOOL" "$LOCAL_TOOL"
  393. scratchpad_test "$LOCAL_PEER_TOOL" "$REMOTE_PEER_TOOL"
  394. scratchpad_test "$REMOTE_PEER_TOOL" "$LOCAL_PEER_TOOL"
  395. message_test "$LOCAL_PEER_TOOL" "$REMOTE_PEER_TOOL"
  396. message_test "$REMOTE_PEER_TOOL" "$LOCAL_PEER_TOOL"
  397. mw_test "$LOCAL_PEER_TOOL" "$REMOTE_PEER_TOOL"
  398. mw_test "$REMOTE_PEER_TOOL" "$LOCAL_PEER_TOOL"
  399. _modprobe -r ntb_tool
  400. }
  401. function ntb_pingpong_tests()
  402. {
  403. LOCAL_PP="$DEBUGFS/ntb_pingpong/$LOCAL_DEV"
  404. REMOTE_PP="$REMOTE_HOST:$DEBUGFS/ntb_pingpong/$REMOTE_DEV"
  405. echo "Starting ntb_pingpong tests..."
  406. _modprobe ntb_pingpong
  407. pingpong_test $LOCAL_PP $REMOTE_PP
  408. _modprobe -r ntb_pingpong
  409. }
  410. function ntb_perf_tests()
  411. {
  412. LOCAL_PERF="$DEBUGFS/ntb_perf/$LOCAL_DEV"
  413. REMOTE_PERF="$REMOTE_HOST:$DEBUGFS/ntb_perf/$REMOTE_DEV"
  414. echo "Starting ntb_perf tests..."
  415. perf_test 0
  416. if [[ $RUN_DMA_TESTS ]]; then
  417. perf_test 1
  418. fi
  419. }
  420. function cleanup()
  421. {
  422. set +e
  423. _modprobe -r ntb_tool 2> /dev/null
  424. _modprobe -r ntb_perf 2> /dev/null
  425. _modprobe -r ntb_pingpong 2> /dev/null
  426. _modprobe -r ntb_transport 2> /dev/null
  427. set -e
  428. }
  429. cleanup
  430. if ! [[ $$DONT_CLEANUP ]]; then
  431. trap cleanup EXIT
  432. fi
  433. if [ "$(id -u)" != "0" ]; then
  434. echo "This script must be run as root" 1>&2
  435. exit 1
  436. fi
  437. if [[ "$LIST_DEVS" == TRUE ]]; then
  438. echo "Local Devices:"
  439. ls -1 /sys/bus/ntb/devices
  440. echo
  441. if [[ "$REMOTE_HOST" != "" ]]; then
  442. echo "Remote Devices:"
  443. ssh $REMOTE_HOST ls -1 /sys/bus/ntb/devices
  444. fi
  445. exit 0
  446. fi
  447. if [[ "$LOCAL_DEV" == $"" ]] || [[ "$REMOTE_DEV" == $"" ]]; then
  448. show_help
  449. exit 1
  450. fi
  451. ntb_tool_tests
  452. echo
  453. ntb_pingpong_tests
  454. echo
  455. ntb_perf_tests
  456. echo