[1818] Run lettuce tests in lockstep, checking availability of services

tests/lettuce/configurations/bindctl_commands.config.orig

+{
+    "version": 2,
+    "Logging": {
+        "loggers": [ {
+            "debuglevel": 99,
+            "severity": "DEBUG",
+            "name": "auth"
+        } ]
+    },
+    "Auth": {
+        "database_file": "data/example.org.sqlite3",
+        "listen_on": [ {
+            "port": 47806,
+            "address": "127.0.0.1"
+        } ]
+    },
+    "Boss": {
+        "components": {
+            "b10-auth": {"kind": "needed", "special": "auth" },
+            "b10-xfrin": { "address": "Xfrin", "kind": "dispensable" },
+            "b10-xfrout": { "address": "Xfrout", "kind": "dispensable" },
+            "b10-zonemgr": { "address": "Zonemgr", "kind": "dispensable" },
+            "b10-stats": { "address": "Stats", "kind": "dispensable" },
+            "b10-stats-httpd": { "address": "StatsHttpd", "kind": "dispensable" },
+            "b10-cmdctl": { "special": "cmdctl", "kind": "needed" }
+        }
+    }
+}

tests/lettuce/configurations/example.org.config.orig

@@ -17,13 +17,7 @@
     "Boss": {
         "components": {
             "b10-auth": {"kind": "needed", "special": "auth" },
-            "b10-xfrin": { "address": "Xfrin", "kind": "dispensable" },
-            "b10-xfrout": { "address": "Xfrout", "kind": "dispensable" },
-            "b10-zonemgr": { "address": "Zonemgr", "kind": "dispensable" },
-            "b10-stats": { "address": "Stats", "kind": "dispensable" },
-            "b10-stats-httpd": { "address": "StatsHttpd", "kind": "dispensable" },
-            "b10-cmdctl": { "special": "cmdctl", "kind": "needed" },
-            "b10-resolver": {"kind": "needed", "special": "resolver"}
+            "b10-cmdctl": { "special": "cmdctl", "kind": "needed" }
         }
     }
 }

tests/lettuce/configurations/example2.org.config

@@ -18,11 +18,6 @@
     "Boss": {
         "components": {
             "b10-auth": {"kind": "needed", "special": "auth" },
-            "b10-xfrin": { "address": "Xfrin", "kind": "dispensable" },
-            "b10-xfrout": { "address": "Xfrout", "kind": "dispensable" },
-            "b10-zonemgr": { "address": "Zonemgr", "kind": "dispensable" },
-            "b10-stats": { "address": "Stats", "kind": "dispensable" },
-            "b10-stats-httpd": { "address": "StatsHttpd", "kind": "dispensable" },
             "b10-cmdctl": { "special": "cmdctl", "kind": "needed" }
         }
     }

tests/lettuce/configurations/no_db_file.config

@@ -10,11 +10,6 @@
     "Boss": {
         "components": {
             "b10-auth": {"kind": "needed", "special": "auth" },
-            "b10-xfrin": { "address": "Xfrin", "kind": "dispensable" },
-            "b10-xfrout": { "address": "Xfrout", "kind": "dispensable" },
-            "b10-zonemgr": { "address": "Zonemgr", "kind": "dispensable" },
-            "b10-stats": { "address": "Stats", "kind": "dispensable" },
-            "b10-stats-httpd": { "address": "StatsHttpd", "kind": "dispensable" },
             "b10-cmdctl": { "special": "cmdctl", "kind": "needed" }
         }
     }

tests/lettuce/features/bindctl_commands.feature

@@ -8,8 +8,9 @@ Feature: control with bindctl
     # modules (note that it 'misuses' the help command for this,
     # there is a Boss command 'show_processes' but it's output is
     # currently less standardized than 'help')
-    Given I have bind10 running with configuration example.org.config
-
+    Given I have bind10 running with configuration bindctl_commands.config
+    And wait for bind10 stderr message BIND10_STARTED_CC
+    And wait for bind10 stderr message CMDCTL_STARTED
     And wait for bind10 stderr message ZONEMGR_STARTED
     And wait for bind10 stderr message AUTH_SERVER_STARTED
     And wait for bind10 stderr message XFRIN_STARTED
@@ -26,31 +27,39 @@ Feature: control with bindctl
     bind10 module Xfrin should be running
     bind10 module Auth should be running
     bind10 module StatsHttpd should be running
+    bind10 module Resolver should not be running
 
     Then remove bind10 configuration Boss/components value b10-xfrout
     And wait for new bind10 stderr message BIND10_PROCESS_ENDED
     last bindctl output should not contain Error
+
     # assuming it won't error for further modules (if it does, the final
     # 'should not be running' tests would fail anyway)
     Then remove bind10 configuration Boss/components value b10-stats-httpd
     And wait for new bind10 stderr message BIND10_PROCESS_ENDED
     last bindctl output should not contain Error
+
     Then remove bind10 configuration Boss/components value b10-stats
     And wait for new bind10 stderr message BIND10_PROCESS_ENDED
     last bindctl output should not contain Error
+
     Then remove bind10 configuration Boss/components value b10-zonemgr
     And wait for new bind10 stderr message BIND10_PROCESS_ENDED
     last bindctl output should not contain Error
+
     Then remove bind10 configuration Boss/components value b10-xfrin
     And wait for new bind10 stderr message BIND10_PROCESS_ENDED
     last bindctl output should not contain Error
+
     Then remove bind10 configuration Boss/components value b10-auth
     And wait for new bind10 stderr message BIND10_PROCESS_ENDED
     last bindctl output should not contain Error
 
+    # After these ^^^ have been stopped...
     bind10 module Xfrout should not be running
     bind10 module Zonemgr should not be running
     bind10 module Xfrin should not be running
     bind10 module Auth should not be running
     bind10 module StatsHttpd should not be running
     bind10 module Stats should not be running
+    bind10 module Resolver should not be running

tests/lettuce/features/default.feature

@@ -3,7 +3,8 @@ Feature: default bind10 config
 
     Scenario: Check that only the default components are running
     Given I have bind10 running with configuration default.config
-
+    And wait for bind10 stderr message BIND10_STARTED_CC
+    And wait for bind10 stderr message CMDCTL_STARTED
     And wait for bind10 stderr message STATS_STARTING
     And wait for bind10 stderr message STATHTTPD_STARTED
 

tests/lettuce/features/example.feature

@@ -8,6 +8,12 @@ Feature: Example feature
     
     Scenario: A simple example
         Given I have bind10 running with configuration example.org.config
+        And wait for bind10 stderr message BIND10_STARTED_CC
+        And wait for bind10 stderr message CMDCTL_STARTED
+        And wait for bind10 stderr message AUTH_SERVER_STARTED
+
+        bind10 module Auth should be running
+
         A query for www.example.org should have rcode NOERROR
         A query for www.doesnotexist.org should have rcode REFUSED
         The SOA serial for example.org should be 1234
@@ -26,8 +32,12 @@ Feature: Example feature
         # is actually a compound step consisting of the following two
         # one to start the server
         When I start bind10 with configuration no_db_file.config
-        # And one to wait until it reports that b10-auth has started
-        Then wait for bind10 auth to start
+
+        And wait for bind10 stderr message BIND10_STARTED_CC
+        And wait for bind10 stderr message CMDCTL_STARTED
+        And wait for bind10 stderr message AUTH_SERVER_STARTED
+
+        bind10 module Auth should be running
 
         # This is a general step to stop a named process. By convention,
         # the default name for any process is the same as the one we
@@ -50,6 +60,11 @@ Feature: Example feature
         # This is a compound statement that starts and waits for the
         # started message
         Given I have bind10 running with configuration example.org.config
+        And wait for bind10 stderr message BIND10_STARTED_CC
+        And wait for bind10 stderr message CMDCTL_STARTED
+        And wait for bind10 stderr message AUTH_SERVER_STARTED
+
+        bind10 module Auth should be running
 
         # Some simple queries that is not examined further
         A query for www.example.com should have rcode REFUSED
@@ -113,8 +128,12 @@ Feature: Example feature
         # the system
 
         When I start bind10 with configuration example.org.config
-        Then wait for bind10 auth to start
-        Wait for bind10 stderr message CMDCTL_STARTED
+        And wait for bind10 stderr message BIND10_STARTED_CC
+        And wait for bind10 stderr message CMDCTL_STARTED
+        And wait for bind10 stderr message AUTH_SERVER_STARTED
+
+        bind10 module Auth should be running
+
         A query for www.example.org should have rcode NOERROR
         Wait for new bind10 stderr message AUTH_SEND_NORMAL_RESPONSE
         Then set bind10 configuration Auth/database_file to data/empty_db.sqlite3
@@ -128,10 +147,17 @@ Feature: Example feature
     Scenario: two bind10 instances
         # This is more a test of the test system, start 2 bind10's
         When I start bind10 with configuration example.org.config as bind10_one
-        And I start bind10 with configuration example2.org.config with cmdctl port 47804 as bind10_two
-
+        And wait for bind10_one stderr message BIND10_STARTED_CC
+        And wait for bind10_one stderr message CMDCTL_STARTED
+        And wait for bind10_one stderr message AUTH_SERVER_STARTED
         Then wait for bind10 auth of bind10_one to start
+
+        And I start bind10 with configuration example2.org.config with cmdctl port 47804 as bind10_two
+        And wait for bind10_two stderr message BIND10_STARTED_CC
+        And wait for bind10_two stderr message CMDCTL_STARTED
+        And wait for bind10_two stderr message AUTH_SERVER_STARTED
         Then wait for bind10 auth of bind10_two to start
+
         A query for www.example.org to 127.0.0.1:47806 should have rcode NOERROR
         A query for www.example.org to [::1]:47807 should have rcode NOERROR
 

tests/lettuce/features/multi_instance.feature

@@ -5,7 +5,17 @@ Feature: Multiple instances
     Scenario: Multiple instances of Auth
         # This config should have two running instances
         Given I have bind10 running with configuration multi_instance/multi_auth.config
+	And wait for bind10 stderr message BIND10_STARTED_CC
+	And wait for bind10 stderr message CMDCTL_STARTED
+	And wait for bind10 stderr message AUTH_SERVER_STARTED
+
         And bind10 module Auth should be running
+        And bind10 module Resolver should not be running
+	bind10 module Xfrout should not be running
+	bind10 module Zonemgr should not be running
+	bind10 module Xfrin should not be running
+	bind10 module Stats should not be running
+
         A query for example.com should have rcode REFUSED
 
         # this also checks whether the process is running
@@ -13,6 +23,7 @@ Feature: Multiple instances
         And remember the pid of process b10-auth-2
 
         When I remove bind10 configuration Boss/components value b10-auth-2
+        And wait for new bind10 stderr message BIND10_PROCESS_ENDED
 
         Then the pid of process b10-auth should not have changed
         And a query for example.com should have rcode REFUSED
@@ -31,5 +42,6 @@ Feature: Multiple instances
         A query for example.com should have rcode REFUSED
 
         When I remove bind10 configuration Boss/components value b10-auth
+        And wait for new bind10 stderr message BIND10_PROCESS_ENDED
         Then the pid of process b10-auth-2 should not have changed
         A query for example.com should have rcode REFUSED

tests/lettuce/features/terrain/terrain.py

@@ -42,6 +42,8 @@ import time
 # The first element is the original, the second is the target that will be
 # used by the tests that need them
 copylist = [
+    ["configurations/bindctl_commands.config.orig",
+     "configurations/bindctl_commands.config"],
     ["configurations/example.org.config.orig",
      "configurations/example.org.config"],
     ["configurations/resolver/resolver_basic.config.orig",