Are --experimental_action_listeners counted against --jobs?

Question

We run clang-tidy as an action listener. It can easily consume as much CPU as compiling gcc, if not more.

Are they written into the -jobs account, or are we planning for the listeners ourselves (for example, when the "correct calibration" of the machine / container / something else)?

Answer 1

Yes they are.

Demo

I run 3 actions + 3 additional actions. Each of them sleeps for 3 seconds and prints when they start and end.

With --jobs=2you can see that leaf2 and leaf3 came together (started at 15:10:00), then list1 and leaf1 listener (at 15:10:03), finally list2 listener and list3 listener (at 15:10:06) .

  $ bazel clean >&/dev/null && time bazel build //:root --jobs=2 --experimental_action_listener="//:clang-tidy-listener" 2>/dev/null
Start action listener at 15:10:03, argc=1, argv=(--extra_action_file=bazel-out/local-fastbuild/extra_actions/clang-tidy/cc4b710d2c96f16eb0bcc5df6f009f08.xa)
Done action listener at 15:10:06
Start action listener at 15:10:06, argc=1, argv=(--extra_action_file=bazel-out/local-fastbuild/extra_actions/clang-tidy/3001ea27a871bf7b111ee4bbbc1d79dc.xa)
Done action listener at 15:10:09
Start action listener at 15:10:06, argc=1, argv=(--extra_action_file=bazel-out/local-fastbuild/extra_actions/clang-tidy/f88e13a970ce3a66354583c477067b29.xa)
Done action listener at 15:10:09

real    0m9.698s
user    0m0.004s
sys     0m0.012s


  $ cat bazel-genfiles/root.txt
leaf1.txt
start: 15:10:03
start: 15:10:06

leaf2.txt
start: 15:10:00
start: 15:10:03

leaf3.txt
start: 15:10:00
start: 15:10:03

--jobs=8 15:08:58:

  $ bazel clean >&/dev/null && time bazel build //:root --jobs=8 --experimental_action_listener="//:clang-tidy-listener" 2>/dev/null
Start action listener at 15:08:58, argc=1, argv=(--extra_action_file=bazel-out/local-fastbuild/extra_actions/clang-tidy/f88e13a970ce3a66354583c477067b29.xa)
Done action listener at 15:09:01
Start action listener at 15:08:58, argc=1, argv=(--extra_action_file=bazel-out/local-fastbuild/extra_actions/clang-tidy/3001ea27a871bf7b111ee4bbbc1d79dc.xa)
Done action listener at 15:09:01
Start action listener at 15:08:58, argc=1, argv=(--extra_action_file=bazel-out/local-fastbuild/extra_actions/clang-tidy/cc4b710d2c96f16eb0bcc5df6f009f08.xa)
Done action listener at 15:09:01

real    0m3.609s
user    0m0.004s
sys     0m0.016s


  $ cat bazel-genfiles/root.txt
leaf1.txt
start: 15:08:58
start: 15:09:01

leaf2.txt
start: 15:08:58
start: 15:09:01

leaf3.txt
start: 15:08:58
start: 15:09:01

//:BUILD:

load(":myrule.bzl", "myrule")

genrule(
    name = "root",
    srcs = [
        "leaf1",
        "leaf2",
        "leaf3",
    ],
    outs = ["root.txt"],
    cmd = "( for f in $(SRCS); do basename $$f ; cat $$f ; echo ; done ; ) > $@",
)

[myrule(name = "leaf%d" % i) for i in [1, 2, 3]]

sh_binary(
    name = "listener",
    srcs = ["listener.sh"],
)

action_listener(
    name = "clang-tidy-listener",
    mnemonics = ["FooAction"],
    extra_actions = [":clang-tidy"],
)

extra_action(
    name = "clang-tidy",
    tools = [":listener"],
    cmd = "$(location :listener) --extra_action_file=$(EXTRA_ACTION_FILE)",
)

//:myrule.bzl:

# Based on https://bazel.build/versions/master/docs/skylark/cookbook.html#simple-shell-command
def _impl(ctx):
  output = ctx.outputs.out
  ctx.action(
       outputs=[output],
       mnemonic="FooAction",
       command="( echo \"start: $(date +%%H:%%M:%%S)\" ; sleep 3 ; echo \"start: $(date +%%H:%%M:%%S)\" ; ) > %s" % output.path)

myrule = rule(
    implementation=_impl,
    outputs={"out": "%{name}.txt"},
)

//:listener.sh:

#!/bin/bash
echo "Start action listener at $(date +%H:%M:%S), argc=$#, argv=($@)"
sleep 3
echo "Done action listener at $(date +%H:%M:%S)"