[benchmarks] run benchmarks twice, for time and resources

We noticed that instrumentation added for watching resources has an
impact in time statistics (i.e. it increases average and stddev). This
commit makes the benchmark tests run twice: once for measuring time and
a second time for measuring resources.

testing/tests/benchmarks/conftest.py

@@ -22,14 +22,6 @@ server.ensure_server()
 # pytest customizations
 #
 
-def pytest_addoption(parser):
-    parser.addoption(
-        "--watch-resources", default=False, action="store_true",
-        help="whether to monitor CPU and memory percentages during test run. "
-             "**Warning**: enabling this will impact the time taken by the "
-             "benchmarked code, so use with caution!")
-
-
 # mark benchmark tests using their group names (thanks ionelmc! :)
 def pytest_collection_modifyitems(items):
     for item in items:

testing/tests/conftest.py

@@ -41,6 +41,14 @@ def pytest_addoption(parser):
         "--couch-url", type="string", default="http://127.0.0.1:5984",
         help="the url for the couch server to be used during tests")
 
+    # the following option is only used in benchmarks, but has to be defined
+    # here due to how pytest discovers plugins during startup.
+    parser.addoption(
+        "--watch-resources", default=False, action="store_true",
+        help="whether to monitor CPU and memory percentages during test run. "
+             "**Warning**: enabling this will impact the time taken by the "
+             "benchmarked code, so use with caution!")
+
 
 @pytest.fixture
 def couch_url(request):

testing/tox.ini

@@ -76,7 +76,9 @@ commands =
 # we must make sure that installed pysqlcipher was built with the HAVE_USLEEP
 # flag, or we might have problems with concurrent db access.
     ./ensure-pysqlcipher-has-usleep.sh
+# run benchmarks twice: once for time and a second time for resources
     py.test --benchmark-only {posargs}
+    py.test --benchmark-only --watch-resources {posargs}
 passenv = HOST_HOSTNAME
 
 [testenv:code-check]