crude tool to find out where all the binary size bloat is coming from and fixes for contbuild: timeout to (try to) avoid hanging tests

Summary: crude tool to find out where all the binary size bloat is coming from
ran it
https://www.facebook.com/groups/fbcode/permalink/930686686968171/

contbuild: timeout to (try to) avoid hanging tests

also the speed test can now have the threshold overridden in the env using WDT_THROUGHPUT  (default 16000 mbytes/sec)

set lower pending https://www.facebook.com/groups/linux.fbk/permalink/1062956390403870/

Reviewed By: nikunjy

Differential Revision: D2631045

fb-gh-sync-id: 97fce9f0bd0f1cc077e836f7e8bea43841b4ea09

Author: ldemailly

build/binary_analyzer.sh

@@ -0,0 +1,56 @@
+#! /bin/sh
+# Quick hack to figure out where is the space coming from
+
+FILE=$1
+if [ -z "$FILE" -o "$FILE" = "-h" ] ; then
+  echo "Usage: $0 binary"
+  exit 1
+fi
+
+TMPRESULT=`mktemp /tmp/nm_out.XXX`
+
+RESULT=$TMPRESULT
+
+echo "Working on $FILE, intermediate nm result in $RESULT"
+
+nm -C --defined-only --print-size -l $FILE > $RESULT
+
+echo "nm done... analyzing..."
+#RESULT=/dev/shm/nm.out
+
+gawk '
+{sizeHex="0x" $2; size=strtonum(sizeHex); sum+=size}
+/:[0-9]+$/ {
+  if (size) {
+    ssum += size;
+    # not all lines of nm output do find a file with line number this finds
+    # lines like    "xxxx/blah.cc:123" and returns "xxxx/blah"
+    # for the xxx part it tries to keep it shorter by not allowing .
+    # (nm sometimes outputs  path/./tmp/morepath)
+    match($0, "([^ \t.]+)(\\.(h|c|cc|tcc|cpp|hpp|S|rl|ipp|y))?:[0-9]+$", arr);
+    src=arr[1]
+    # unknown extensions...
+    if (length(src) < 5) print src, "  XXXX ", $0;
+    # get rid of irrelevant stuff
+    sub("/tmp/", "/", src)
+    # somehow lots of ....fbcode...////////morestuf
+    sub(".*///", "", src)
+    # x/ or y/ ...
+    sub("^./", "", src)
+    # 20/....
+    sub("^[0-9/]+", "", src)
+    sizes[src] += size;
+    match(src, "(.*)/[^/]*$", pkg);
+    sizes[pkg[1] "/*"]+= size;
+  }
+}
+END {
+  print "Extracted file attribution for", ssum/1024/1024., "Mb out of",
+     sum/1024/1024., "Mb :", int((1000*ssum/sum+.5))/10, "%"
+  n = asorti(sizes, siter)
+  for (s in sizes) {
+    print s, sizes[s]
+  }
+}' $RESULT | sort -k 2 -n
+
+rm $TMPRESULT

build/wdt_cont_build_run.tcl

@@ -10,6 +10,15 @@ append ::env(USER) "_wdt_contbuild"
 set userdir $::env(USER)
 puts "Will run script with USER env = $userdir"
 
+set maxTestDuration "15m"
+set totalMaxDuration "35m"
+
+puts "Max test duration: $maxTestDuration - Max total $totalMaxDuration"
+
+# Set throughput - lower for now / there is some issue with kernel or env
+# (or our code?)
+set ::env(WDT_THROUGHPUT) 13000
+
 set CDIR "/data/users/$userdir"
 
 # path and ld library path
@@ -43,6 +52,8 @@ proc sleep {time} {
 
 # we will email for the first change
 set last {}
+# uncomment to force initial version update after restart
+# set last "force"
 # previous hg log for wdt (will cause email first too)
 set hgprev {none}
 # also email every x :
@@ -80,15 +91,18 @@ nextEmail
 
 cd $CDIR/fbsource/fbcode
 
+
 # only 2 types for now - either 'open source' on the mac or full otherwise
 set os [exec uname]
 if {$os == "Darwin"} {
     set type "mac"
     set extraCmds "echo done"
     set targetDir "/usr/local/var/www/wdt_builds/"
     set sudo ""
+    set timeoutCmd "gtimeout"
 } else {
     set type "unix"
+    set timeoutCmd "timeout"
     set extraCmds "cd $CDIR/fbsource/fbcode &&\
      (sudo tc qdisc del dev lo root; sudo ip6tables --flush || true) &&\
      time fbconfig --clang -r wdt &&\
@@ -97,11 +111,11 @@ if {$os == "Darwin"} {
      time wdt/test/wdt_max_send_test.sh _bin/wdt/fbonly/wdt_fb |& tail -50 &&\
      time fbconfig --sanitize address -r wdt &&\
      time fbmake dbg &&\
-     time fbmake runtests --run-disabled --record-results --return-nonzero-on-timeouts &&\
+     time $timeoutCmd $maxTestDuration fbmake runtests --run-disabled --record-results --return-nonzero-on-timeouts &&\
      sudo tc qdisc add dev lo root netem delay 20ms 10ms \
      duplicate 1% corrupt 0.1% &&\
      echo rerunning tests with tc delays &&\
-     time fbmake runtests --run-disabled --record-results --return-nonzero-on-timeouts &&\
+     time $timeoutCmd $maxTestDuration fbmake runtests --run-disabled --record-results --return-nonzero-on-timeouts &&\
      sudo tc qdisc del dev lo root"
     set targetDir "~/public_html/wdt_builds/"
     set sudo "sudo"
@@ -122,12 +136,13 @@ while {1} {
     set LOGF "$CDIR/$LOGTS.log"
     puts "Logging to $LOGF"
     # cleanup previous builds failure - sudo not needed/asking for passwd on mac
-    if {[catch {exec sh -c "set -o pipefail; set -x; date; uname -a;\
+    if {[catch {exec $timeoutCmd $totalMaxDuration sh -c "set -o pipefail;\
+     set -x; date; uname -a;\
      $sudo rm -rf /tmp/wdtTest_$userdir /dev/shm/wdtTest_$userdir wdtTest &&\
      cd $CDIR/fbsource/fbcode && time hg pull -u &&\
      hg log -l 1 && hg log -v -l 1 folly && hg log -v -l 1 wdt &&\
      cd $CDIR/cmake_wdt_build && time make -j 4 && \
-     CTEST_OUTPUT_ON_FAILURE=1 time make test &&\
+     CTEST_OUTPUT_ON_FAILURE=1 time $timeoutCmd $maxTestDuration make test &&\
      $extraCmds" >& $LOGF < /dev/null} results options]} {
         set msg "BAD"
         set good 0

test/wdt_max_send_test.sh

@@ -34,6 +34,12 @@ if [ -z $TEST_COUNT ]; then
   TEST_COUNT=16
 fi
 
+# WDT_THROUGHPUT is an env var. Set it to overwrite the expected 16Gbyte/sec
+if [ -z $WDT_THROUGHPUT ]; then
+  WDT_THROUGHPUT=16000
+fi
+
+
 if [ -z "$HOSTNAME" ] ; then
     echo "HOSTNAME not set, will try with 'localhost'"
     HOSTNAME=localhost
@@ -143,8 +149,8 @@ rm -rf $DIR
 echo "Speed for all runs: $ALLSPEEDS"
 
 # Normalize by CPU / number of threads
-# (32 cores leaves 15 threads and can do ~20-21G, use 19 for margin of noise)
-EXPECTED_SPEED=`echo 16000*$NUM_THREADS/16|bc`
+# (32 cores leaves 15 threads and can do ~20-22G, use 16 for margin of noise)
+EXPECTED_SPEED=`echo $WDT_THROUGHPUT*$NUM_THREADS/16|bc`
 
 echo "Best throughput for $WDTBIN"
 if [ "$MAXRATE" -lt "$EXPECTED_SPEED" ]; then