Merge #2439

Perform post transfer validation of zone in XfrIn by isc.dn.check_zone and
reject broken zones.

src/bin/xfrin/tests/xfrin_test.py

@@ -153,13 +153,19 @@ class MockCC(MockModuleCCSession):
     def remove_remote_config(self, module_name):
         pass
 
+class MockRRsetCollection:
+    '''
+    A mock RRset collection. We don't use it really (we mock the method that
+    it is passed to too), so it's empty.
+    '''
+    pass
+
 class MockDataSourceClient():
     '''A simple mock data source client.
 
     This class provides a minimal set of wrappers related the data source
     API that would be used by Diff objects.  For our testing purposes they
-    only keep truck of the history of the changes.
-
+    only keep track of the history of the changes.
     '''
     def __init__(self):
         self.force_fail = False # if True, raise an exception on commit
@@ -217,6 +223,12 @@ class MockDataSourceClient():
         self._journaling_enabled = journaling
         return self
 
+    def get_rrset_collection(self):
+        '''
+        Pretend to be a zone updater and provide a (dummy) rrset collection.
+        '''
+        return MockRRsetCollection()
+
     def add_rrset(self, rrset):
         self.diffs.append(('add', rrset))
 
@@ -726,11 +738,27 @@ class TestXfrinConnection(unittest.TestCase):
             'tsig_1st': None,
             'tsig_2nd': None
             }
+        self.__orig_check_zone = xfrin.check_zone
+        xfrin.check_zone = self.__check_zone
+        self._check_zone_result = True
+        self._check_zone_params = None
 
     def tearDown(self):
         self.conn.close()
         if os.path.exists(TEST_DB_FILE):
             os.remove(TEST_DB_FILE)
+        xfrin.check_zone = self.__orig_check_zone
+
+    def __check_zone(self, name, rrclass, rrsets, callbacks):
+        '''
+        A mock function used instead of dns.check_zone.
+        '''
+        self._check_zone_params = (name, rrclass, rrsets, callbacks)
+        # Call both callbacks to see they do nothing. This checks
+        # the transfer depends on the result only.
+        callbacks[0]("Test error")
+        callbacks[1]("Test warning")
+        return self._check_zone_result
 
     def _create_normal_response_data(self):
         # This helper method creates a simple sequence of DNS messages that
@@ -825,6 +853,7 @@ class TestAXFR(TestXfrinConnection):
 
     def tearDown(self):
         time.time = self.orig_time_time
+        super().tearDown()
 
     def __create_mock_tsig(self, key, error, has_last_signature=True):
         # This helper function creates a MockTSIGContext for a given key
@@ -1297,6 +1326,33 @@ class TestAXFR(TestXfrinConnection):
                     [[('add', ns_rr), ('add', a_rr), ('add', soa_rrset)]],
                     self.conn._datasrc_client.committed_diffs)
 
+    def test_axfr_response_fail_validation(self):
+        """
+        Test we reject a zone transfer if it fails the check_zone validation.
+        """
+        a_rr = self._create_a('192.0.2.1')
+        self.conn._send_query(RRType.AXFR())
+        self.conn.reply_data = self.conn.create_response_data(
+            questions=[Question(TEST_ZONE_NAME, TEST_RRCLASS,
+                                RRType.AXFR())],
+            # begin serial=1230, end serial=1234. end will be used.
+            answers=[begin_soa_rrset, a_rr, soa_rrset])
+        # Make it fail the validation
+        self._check_zone_result = False
+        self.assertRaises(XfrinZoneError, self.conn._handle_xfrin_responses)
+        self.assertEqual(type(XfrinAXFREnd()), type(self.conn.get_xfrstate()))
+        self.assertEqual([], self.conn._datasrc_client.committed_diffs)
+        # Check the validation is called with the correct parameters
+        self.assertEqual(TEST_ZONE_NAME, self._check_zone_params[0])
+        self.assertEqual(TEST_RRCLASS, self._check_zone_params[1])
+        self.assertTrue(isinstance(self._check_zone_params[2],
+                                   MockRRsetCollection))
+        # Check we can safely call the callbacks. They have no sideeffects
+        # we can check (checking logging is hard), but we at least check
+        # they don't crash.
+        self._check_zone_params[3][0]("Test error")
+        self._check_zone_params[3][1]("Test warning")
+
     def test_axfr_response_extra(self):
         '''Test with an extra RR after the end of AXFR session.
 
@@ -1499,6 +1555,15 @@ class TestAXFR(TestXfrinConnection):
         self.conn.response_generator = self._create_normal_response_data
         self.assertEqual(self.conn.do_xfrin(False), XFRIN_FAIL)
 
+    def test_do_xfrin_invalid_zone(self):
+        """
+        Test receiving an invalid zone. We mock the check and see the whole
+        transfer is rejected.
+        """
+        self._check_zone_result = False
+        self.conn.response_generator = self._create_normal_response_data
+        self.assertEqual(self.conn.do_xfrin(False), XFRIN_FAIL)
+
     def test_do_soacheck_and_xfrin(self):
         self.conn.response_generator = self._create_soa_response_data
         self.assertEqual(self.conn.do_xfrin(True), XFRIN_OK)
@@ -1576,6 +1641,26 @@ class TestIXFRResponse(TestXfrinConnection):
                     [[('delete', begin_soa_rrset), ('add', soa_rrset)]],
                     self.conn._datasrc_client.committed_diffs)
 
+    def test_ixfr_response_fail_validation(self):
+        '''
+        An IXFR that fails validation later on. Check it is rejected.
+        '''
+        self.conn.reply_data = self.conn.create_response_data(
+            questions=[Question(TEST_ZONE_NAME, TEST_RRCLASS, RRType.IXFR())],
+            answers=[soa_rrset, begin_soa_rrset, soa_rrset, soa_rrset])
+        self._check_zone_result = False
+        self.assertRaises(XfrinZoneError, self.conn._handle_xfrin_responses)
+        self.assertEqual([], self.conn._datasrc_client.committed_diffs)
+        self.assertEqual(TEST_ZONE_NAME, self._check_zone_params[0])
+        self.assertEqual(TEST_RRCLASS, self._check_zone_params[1])
+        self.assertTrue(isinstance(self._check_zone_params[2],
+                                   MockRRsetCollection))
+        # Check we can safely call the callbacks. They have no sideeffects
+        # we can check (checking logging is hard), but we at least check
+        # they don't crash.
+        self._check_zone_params[3][0]("Test error")
+        self._check_zone_params[3][1]("Test warning")
+
     def test_ixfr_response_multi_sequences(self):
         '''Similar to the previous case, but with multiple diff seqs.
 

src/bin/xfrin/xfrin.py.in

@@ -36,6 +36,7 @@ from isc.xfrin.diff import Diff
 from isc.server_common.auth_command import auth_loadzone_command
 from isc.server_common.tsig_keyring import init_keyring, get_keyring
 from isc.log_messages.xfrin_messages import *
+from isc.dns import *
 
 isc.log.init("b10-xfrin", buffer=True)
 logger = isc.log.Logger("xfrin")
@@ -45,13 +46,6 @@ logger = isc.log.Logger("xfrin")
 DBG_PROCESS = logger.DBGLVL_TRACE_BASIC
 DBG_COMMANDS = logger.DBGLVL_TRACE_DETAIL
 
-try:
-    from pydnspp import *
-except ImportError as e:
-    # C++ loadable module may not be installed; even so the xfrin process
-    # must keep running, so we warn about it and move forward.
-    logger.error(XFRIN_IMPORT_DNS, str(e))
-
 isc.util.process.rename()
 
 # If B10_FROM_BUILD is set in the environment, we use data files
@@ -100,8 +94,17 @@ class XfrinProtocolError(Exception):
     '''
     pass
 
+class XfrinZoneError(Exception):
+    '''
+    An exception raised when the received zone is broken enough to be unusable.
+    '''
+    pass
+
 class XfrinZoneUptodate(Exception):
-    '''TBD
+    '''
+    Thrown when the zone is already up to date, so there's no need to download
+    the zone. This is not really an error case (but it's still an exceptional
+    condition and the control flow is different than usual).
     '''
     pass
 
@@ -427,10 +430,10 @@ class XfrinIXFRAdd(XfrinState):
             conn.get_transfer_stats().ixfr_changeset_count += 1
             soa_serial = get_soa_serial(rr.get_rdata()[0])
             if soa_serial == conn._end_serial:
-                # The final part is there. Check all was signed
-                # and commit it to the database.
-                conn._check_response_tsig_last()
-                conn._diff.commit()
+                # The final part is there. Finish the transfer by
+                # checking the last TSIG (if required), the zone data and
+                # commiting.
+                conn.finish_transfer()
                 self.set_xfrstate(conn, XfrinIXFREnd())
                 return True
             elif soa_serial != conn._current_serial:
@@ -500,15 +503,11 @@ class XfrinAXFREnd(XfrinState):
         """
         Final processing after processing an entire AXFR session.
 
-        In this process all the AXFR changes are committed to the
-        data source.
-
-        There might be more actions here, but for now we simply return False,
-        indicating there will be no more message to receive.
-
+        This simply calls the finish_transfer method of the connection
+        that ensures it is signed by TSIG (if required), the zone data
+        is valid and commits it.
         """
-        conn._check_response_tsig_last()
-        conn._diff.commit()
+        conn.finish_transfer()
         return False
 
 class XfrinTransferStats:
@@ -805,6 +804,31 @@ class XfrinConnection(asyncore.dispatcher):
                 raise XfrinProtocolError('TSIG verify fail: no TSIG on last '+
                                          'message')
 
+    def __validate_error(self, reason):
+        '''
+        Used as error callback below.
+        '''
+        logger.error(XFRIN_ZONE_INVALID, self._zone_name, self._rrclass,
+                     reason)
+
+    def __validate_warning(self, reason):
+        '''
+        Used as warning callback below.
+        '''
+        logger.warn(XFRIN_ZONE_WARN, self._zone_name, self._rrclass, reason)
+
+    def finish_transfer(self):
+        """
+        Perform any necessary checks after a transfer. Then complete the
+        transfer by commiting the transaction into the data source.
+        """
+        self._check_response_tsig_last()
+        if not check_zone(self._zone_name, self._rrclass,
+                          self._diff.get_rrset_collection(),
+                          (self.__validate_error, self.__validate_warning)):
+            raise XfrinZoneError('Validation of the new zone failed')
+        self._diff.commit()
+
     def __parse_soa_response(self, msg, response_data):
         '''Parse a response to SOA query and extract the SOA from answer.
 
@@ -950,7 +974,15 @@ class XfrinConnection(asyncore.dispatcher):
             # of trying another primary server, etc, but for now we treat it
             # as "success".
             pass
+        except XfrinZoneError:
+            # The log message doesn't contain the exception text, since there's
+            # only one place where the exception is thrown now and it'd be the
+            # same generic message every time.
+            logger.error(XFRIN_INVALID_ZONE_DATA, self.zone_str(),
+                         format_addrinfo(self._master_addrinfo))
+            ret = XFRIN_FAIL
         except XfrinProtocolError as e:
+            # FIXME: Why is this .info? Even the messageID contains "ERROR".
             logger.info(XFRIN_XFR_TRANSFER_PROTOCOL_ERROR, req_str,
                         self.zone_str(),
                         format_addrinfo(self._master_addrinfo), str(e))

src/bin/xfrin/xfrin_messages.mes

@@ -77,6 +77,11 @@ is not equal to the requested SOA serial.
 There was an error importing the python DNS module pydnspp. The most
 likely cause is a PYTHONPATH problem.
 
+% XFRIN_INVALID_ZONE_DATA zone %1 received from %2 is broken and unusable
+The zone was received, but it failed sanity validation. The previous version
+of zone (if any is available) will be used. Look for previous
+XFRIN_ZONE_INVALID messages to see the exact problem(s).
+
 % XFRIN_IXFR_TRANSFER_SUCCESS incremental IXFR transfer of zone %1 succeeded (messages: %2, changesets: %3, deletions: %4, additions: %5, bytes: %6, run time: %7 seconds, %8 bytes/second)
 The IXFR transfer for the given zone was successful.
 The provided information contains the following values:
@@ -232,6 +237,12 @@ zones at a higher level.  In future it is more likely that a separate
 zone management framework is provided, and the situation where the
 given zone isn't found in xfrout will be treated as an error.
 
+% XFRIN_ZONE_INVALID Newly received zone %1/%2 fails validation: %3
+The zone was received successfully, but it failed validation. The problem
+is severe enough that the new version of zone is discarded and the old version,
+if any, will stay in use. New transfer will be attempted after some time.
+The problem needs to be fixed in the zone data on the remote server.
+
 % XFRIN_ZONE_MULTIPLE_SOA Zone %1 has %2 SOA RRs
 On starting an xfrin session, it is identified that the zone to be
 transferred has multiple SOA RRs.  Such a zone is broken, but could be
@@ -258,3 +269,9 @@ the latest version of the zone.  But if the primary server is known to
 be the real source of the zone, some unexpected inconsistency may have
 happened, and you may want to take a closer look.  In this case xfrin
 doesn't perform subsequent zone transfer.
+
+% XFRIN_ZONE_WARN Newly received zone %1/%2 has a problem: %3
+The zone was received successfully, but when checking it, it was discovered
+there's some issue with it. It might be correct, but it should be checked
+and possibly fixed on the remote server. The problem is described in the
+message. The problem does not stop the zone from being used.

src/lib/python/isc/xfrin/diff.py

@@ -584,3 +584,16 @@ class Diff:
             if rr.get_name() == name:
                 new_rrsets.append(rr)
         return result, new_rrsets, flags
+
+    def get_rrset_collection(self):
+        '''
+        This first applies all changes to the data source. Then it creates
+        and returns an RRsetCollection on top of the corresponding zone
+        updater. Notice it might be impossible to apply more changes after
+        that.
+
+        This must not be called after a commit, or it'd throw ValueError.
+        '''
+        # Apply itself will check it is not yet commited.
+        self.apply()
+        return self.__updater.get_rrset_collection()

src/lib/python/isc/xfrin/tests/diff_tests.py

@@ -16,7 +16,8 @@
 import isc.log
 import unittest
 from isc.datasrc import ZoneFinder
-from isc.dns import Name, RRset, RRClass, RRType, RRTTL, Rdata
+from isc.dns import Name, RRset, RRClass, RRType, RRTTL, Rdata, \
+    RRsetCollectionBase
 from isc.xfrin.diff import Diff, NoSuchZone
 
 class TestError(Exception):
@@ -1087,6 +1088,52 @@ class DiffTest(unittest.TestCase):
             self.__check_find_all_call(diff.find_all, self.__rrset3,
                                        rcode)
 
+    class Collection(isc.dns.RRsetCollectionBase):
+        '''
+        Our own mock RRsetCollection. We only pass it through, but we
+        still define an (mostly empty) find method to satisfy the
+        expectations.
+        '''
+        def __init__(self):
+            '''
+            Empty init. The base class's __init__ can't be called,
+            so we need to provide our own to shadow it -- and make sure
+            not to call the super().__init__().
+            '''
+            pass
+
+        def find(self, name, rrclass, rrtype):
+            '''
+            Empty find method. Returns None to each query (pretends
+            the collection is empty. Present mostly for completeness.
+            '''
+            return None
+
+    def get_rrset_collection(self):
+        '''
+        Part of pretending to be the zone updater. This returns the rrset
+        collection (a mock one, unuseable) for the updater.
+        '''
+        return self.Collection()
+
+    def test_get_rrset_collection(self):
+        '''
+        Test the diff can return corresponding rrset collection. Test
+        it applies the data first.
+        '''
+        diff = Diff(self, Name('example.org'), single_update_mode=True)
+        diff.add_data(self.__rrset_soa)
+        collection = diff.get_rrset_collection()
+        # Check it is applied
+        self.assertEqual(1, len(self.__data_operations))
+        self.assertEqual('add', self.__data_operations[0][0])
+        # Check the returned one is actually RRsetCollection
+        self.assertTrue(isinstance(collection, self.Collection))
+        # The collection is just the mock from above, so this doesn't do much
+        # testing, but we check that the mock got through and didn't get hurt.
+        self.assertIsNone(collection.find(Name('example.org'), RRClass.IN(),
+                                          RRType.SOA()))
+
 if __name__ == "__main__":
     isc.log.init("bind10")
     isc.log.resetUnitTestRootLogger()

tests/lettuce/configurations/xfrin/retransfer_master_nons.conf.orig

+{
+    "version": 2,
+    "Logging": {
+        "loggers": [ {
+            "debuglevel": 99,
+            "severity": "DEBUG",
+            "name": "*"
+        } ]
+    },
+    "Auth": {
+        "database_file": "data/example.org-nons.sqlite3",
+        "listen_on": [ {
+            "address": "::1",
+            "port": 47807
+        } ]
+    },
+    "data_sources": {
+        "classes": {
+            "IN": [{
+                "type": "sqlite3",
+                "params": {
+                    "database_file": "data/example.org-nons.sqlite3"
+                }
+            }]
+        }
+    },
+    "Xfrout": {
+        "zone_config": [ {
+            "origin": "example.org"
+        } ],
+        "also_notify": [ {
+            "address": "::1",
+            "port": 47806
+        } ]
+    },
+    "Stats": {
+        "poll-interval": 1
+    },
+    "Boss": {
+        "components": {
+            "b10-auth": { "kind": "needed", "special": "auth" },
+            "b10-xfrout": { "address": "Xfrout", "kind": "dispensable" },
+            "b10-zonemgr": { "address": "Zonemgr", "kind": "dispensable" },
+            "b10-stats": { "address": "Stats", "kind": "dispensable" },
+            "b10-cmdctl": { "special": "cmdctl", "kind": "needed" }
+        }
+    }
+}

tests/lettuce/features/terrain/terrain.py

@@ -64,6 +64,8 @@ copylist = [
      "configurations/ddns/noddns.config"],
     ["configurations/xfrin/retransfer_master.conf.orig",
      "configurations/xfrin/retransfer_master.conf"],
+    ["configurations/xfrin/retransfer_master_nons.conf.orig",
+     "configurations/xfrin/retransfer_master_nons.conf"],
     ["configurations/xfrin/retransfer_slave.conf.orig",
      "configurations/xfrin/retransfer_slave.conf"],
     ["data/inmem-xfrin.sqlite3.orig",

tests/lettuce/features/xfrin_bind10.feature

@@ -25,6 +25,13 @@ Feature: Xfrin
 
     A query for www.example.org to [::1]:47806 should have rcode REFUSED
     When I send bind10 the command Xfrin retransfer example.org IN ::1 47807
+    # The data we receive contain a NS RRset that refers to three names in the
+    # example.org. zone. All these three are nonexistent in the data, producing
+    # 3 separate warning messages in the log.
+    And wait for new bind10 stderr message XFRIN_ZONE_WARN
+    And wait for new bind10 stderr message XFRIN_ZONE_WARN
+    And wait for new bind10 stderr message XFRIN_ZONE_WARN
+    # But after complaining, the zone data should be accepted.
     Then wait for new bind10 stderr message XFRIN_TRANSFER_SUCCESS not XFRIN_XFR_PROCESS_FAILURE
     Then wait for new bind10 stderr message ZONEMGR_RECEIVE_XFRIN_SUCCESS
     A query for www.example.org to [::1]:47806 should have rcode NOERROR
@@ -38,7 +45,20 @@ Feature: Xfrin
     When I do an AXFR transfer of example.org
     Then transfer result should have 13 rrs
 
-
+    # Now try to offer another update. However, the validation of
+    # data should fail. The old version shoud still be available.
+    When I send bind10 the following commands with cmdctl port 47804:
+    """
+    config set data_sources/classes/IN[0]/params/database_file data/example.org-nons.sqlite3
+    config set Auth/database_file data/example.org-nons.sqlite3
+    config commit
+    """
+    Then I send bind10 the command Xfrin retransfer example.org IN ::1 47807
+    And wait for new bind10 stderr message XFRIN_ZONE_INVALID
+    And wait for new bind10 stderr message XFRIN_INVALID_ZONE_DATA
+    Then wait for new bind10 stderr message ZONEMGR_RECEIVE_XFRIN_FAILED
+    A query for example.org type NS to [::1]:47806 should have rcode NOERROR
+    And transfer result should have 13 rrs
 
     Scenario: Transfer with TSIG
     # Similar setup to the test above, but this time, we add TSIG configuration
@@ -86,3 +106,41 @@ Feature: Xfrin
     # Transwer should succeed now
     When I send bind10 the command Xfrin retransfer example.org
     Then wait for new bind10 stderr message XFRIN_TRANSFER_SUCCESS not XFRIN_XFR_PROCESS_FAILURE
+
+    Scenario: Validation fails
+    # In this test, the source data of the XFR is invalid (missing NS record
+    # at the origin). We check it is rejected after the transfer.
+    #
+    # We use abuse the fact that we do not check data when we read it from
+    # the sqlite3 database (unless we load into in-memory, which we don't
+    # do here).
+    The file data/test_nonexistent_db.sqlite3 should not exist
+
+    Given I have bind10 running with configuration xfrin/retransfer_master_nons.conf with cmdctl port 47804 as master
+    And wait for master stderr message BIND10_STARTED_CC
+    And wait for master stderr message CMDCTL_STARTED
+    And wait for master stderr message AUTH_SERVER_STARTED
+    And wait for master stderr message XFROUT_STARTED
+    And wait for master stderr message ZONEMGR_STARTED
+
+    And I have bind10 running with configuration xfrin/retransfer_slave.conf
+    And wait for bind10 stderr message BIND10_STARTED_CC
+    And wait for bind10 stderr message CMDCTL_STARTED
+    And wait for bind10 stderr message AUTH_SERVER_STARTED
+    And wait for bind10 stderr message XFRIN_STARTED
+    And wait for bind10 stderr message ZONEMGR_STARTED
+
+    # Now we use the first step again to see if the file has been created
+    The file data/test_nonexistent_db.sqlite3 should exist
+
+    A query for www.example.org to [::1]:47806 should have rcode REFUSED
+    When I send bind10 the command Xfrin retransfer example.org IN ::1 47807
+    # It should complain once about invalid data, then again that the whole
+    # zone is invalid and then reject it.
+    And wait for new bind10 stderr message XFRIN_ZONE_INVALID
+    And wait for new bind10 stderr message XFRIN_INVALID_ZONE_DATA
+    Then wait for new bind10 stderr message ZONEMGR_RECEIVE_XFRIN_FAILED
+    # The zone still doesn't exist as it is rejected.
+    # FIXME: This step fails. Probably an empty zone is created in the data
+    # source :-|. This should be REFUSED, not SERVFAIL.
+    A query for www.example.org to [::1]:47806 should have rcode SERVFAIL