Commit a5eeb731 authored by Michal 'vorner' Vaner's avatar Michal 'vorner' Vaner
Browse files

Merge #1259

parents eb4917ae 743dad94
......@@ -853,6 +853,8 @@ AC_CONFIG_FILES([Makefile
src/lib/python/isc/testutils/Makefile
src/lib/python/isc/bind10/Makefile
src/lib/python/isc/bind10/tests/Makefile
src/lib/python/isc/xfrin/Makefile
src/lib/python/isc/xfrin/tests/Makefile
src/lib/config/Makefile
src/lib/config/tests/Makefile
src/lib/config/tests/testdata/Makefile
......
SUBDIRS = datasrc cc config dns log net notify util testutils acl bind10
SUBDIRS += log_messages
SUBDIRS += xfrin log_messages
python_PYTHON = __init__.py
......
......@@ -11,6 +11,7 @@ EXTRA_DIST += zonemgr_messages.py
EXTRA_DIST += cfgmgr_messages.py
EXTRA_DIST += config_messages.py
EXTRA_DIST += notify_out_messages.py
EXTRA_DIST += libxfrin_messages.py
CLEANFILES = __init__.pyc
CLEANFILES += bind10_messages.pyc
......@@ -23,6 +24,7 @@ CLEANFILES += zonemgr_messages.pyc
CLEANFILES += cfgmgr_messages.pyc
CLEANFILES += config_messages.pyc
CLEANFILES += notify_out_messages.pyc
CLEANFILES += libxfrin_messages.pyc
CLEANDIRS = __pycache__
......
from work.libxfrin_messages import *
SUBDIRS = . tests
python_PYTHON = __init__.py diff.py
BUILT_SOURCES = $(PYTHON_LOGMSGPKG_DIR)/work/libxfrin_messages.py
nodist_pylogmessage_PYTHON = $(PYTHON_LOGMSGPKG_DIR)/work/libxfrin_messages.py
pylogmessagedir = $(pyexecdir)/isc/log_messages/
EXTRA_DIST = libxfrin_messages.mes
CLEANFILES = $(PYTHON_LOGMSGPKG_DIR)/work/libxfrin_messages.py
CLEANFILES += $(PYTHON_LOGMSGPKG_DIR)/work/libxfrin_messages.pyc
# Define rule to build logging source files from message file
$(PYTHON_LOGMSGPKG_DIR)/work/libxfrin_messages.py: libxfrin_messages.mes
$(top_builddir)/src/lib/log/compiler/message \
-d $(PYTHON_LOGMSGPKG_DIR)/work -p $(srcdir)/libxfrin_messages.mes
pythondir = $(pyexecdir)/isc/xfrin
CLEANDIRS = __pycache__
clean-local:
rm -rf $(CLEANDIRS)
# Copyright (C) 2011 Internet Systems Consortium.
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM
# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT,
# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
"""
This helps the XFR in process with accumulating parts of diff and applying
it to the datasource.
The name of the module is not yet fully decided. We might want to move it
under isc.datasrc or somewhere else, because we might want to reuse it with
future DDNS process. But until then, it lives here.
"""
import isc.dns
import isc.log
from isc.log_messages.libxfrin_messages import *
class NoSuchZone(Exception):
"""
This is raised if a diff for non-existant zone is being created.
"""
pass
"""
This is the amount of changes we accumulate before calling Diff.apply
automatically.
The number 100 is just taken from BIND 9. We don't know the rationale
for exactly this amount, but we think it is just some randomly chosen
number.
"""
# If changing this, modify the tests accordingly as well.
DIFF_APPLY_TRESHOLD = 100
logger = isc.log.Logger('libxfrin')
class Diff:
"""
The class represents a diff against current state of datasource on
one zone. The usual way of working with it is creating it, then putting
bunch of changes in and commiting at the end.
If you change your mind, you can just stop using the object without
really commiting it. In that case no changes will happen in the data
sounce.
The class works as a kind of a buffer as well, it does not direct
the changes to underlying data source right away, but keeps them for
a while.
"""
def __init__(self, ds_client, zone):
"""
Initializes the diff to a ready state. It checks the zone exists
in the datasource and if not, NoSuchZone is raised. This also creates
a transaction in the data source.
The ds_client is the datasource client containing the zone. Zone is
isc.dns.Name object representing the name of the zone (its apex).
You can also expect isc.datasrc.Error or isc.datasrc.NotImplemented
exceptions.
"""
self.__updater = ds_client.get_updater(zone, False)
if self.__updater is None:
# The no such zone case
raise NoSuchZone("Zone " + str(zone) +
" does not exist in the data source " +
str(ds_client))
self.__buffer = []
def __check_commited(self):
"""
This checks if the diff is already commited or broken. If it is, it
raises ValueError. This check is for methods that need to work only on
yet uncommited diffs.
"""
if self.__updater is None:
raise ValueError("The diff is already commited or it has raised " +
"an exception, you come late")
def __data_common(self, rr, operation):
"""
Schedules an operation with rr.
It does all the real work of add_data and remove_data, including
all checks.
"""
self.__check_commited()
if rr.get_rdata_count() != 1:
raise ValueError('The rrset must contain exactly 1 Rdata, but ' +
'it holds ' + str(rr.get_rdata_count()))
if rr.get_class() != self.__updater.get_class():
raise ValueError("The rrset's class " + str(rr.get_class()) +
" does not match updater's " +
str(self.__updater.get_class()))
self.__buffer.append((operation, rr))
if len(self.__buffer) >= DIFF_APPLY_TRESHOLD:
# Time to auto-apply, so the data don't accumulate too much
self.apply()
def add_data(self, rr):
"""
Schedules addition of an RR into the zone in this diff.
The rr is of isc.dns.RRset type and it must contain only one RR.
If this is not the case or if the diff was already commited, this
raises the ValueError exception.
The rr class must match the one of the datasource client. If
it does not, ValueError is raised.
"""
self.__data_common(rr, 'add')
def remove_data(self, rr):
"""
Schedules removal of an RR from the zone in this diff.
The rr is of isc.dns.RRset type and it must contain only one RR.
If this is not the case or if the diff was already commited, this
raises the ValueError exception.
The rr class must match the one of the datasource client. If
it does not, ValueError is raised.
"""
self.__data_common(rr, 'remove')
def compact(self):
"""
Tries to compact the operations in buffer a little by putting some of
the operations together, forming RRsets with more than one RR.
This is called by apply before putting the data into datasource. You
may, but not have to, call this manually.
Currently it merges consecutive same operations on the same
domain/type. We could do more fancy things, like sorting by the domain
and do more merging, but such diffs should be rare in practice anyway,
so we don't bother and do it this simple way.
"""
buf = []
for (op, rrset) in self.__buffer:
old = buf[-1][1] if len(buf) > 0 else None
if old is None or op != buf[-1][0] or \
rrset.get_name() != old.get_name() or \
rrset.get_type() != old.get_type():
buf.append((op, isc.dns.RRset(rrset.get_name(),
rrset.get_class(),
rrset.get_type(),
rrset.get_ttl())))
if rrset.get_ttl() != buf[-1][1].get_ttl():
logger.warn(LIBXFRIN_DIFFERENT_TTL, rrset.get_ttl(),
buf[-1][1].get_ttl())
for rdatum in rrset.get_rdata():
buf[-1][1].add_rdata(rdatum)
self.__buffer = buf
def apply(self):
"""
Push the buffered changes inside this diff down into the data source.
This does not stop you from adding more changes later through this
diff and it does not close the datasource transaction, so the changes
will not be shown to others yet. It just means the internal memory
buffer is flushed.
This is called from time to time automatically, but you can call it
manually if you really want to.
This raises ValueError if the diff was already commited.
It also can raise isc.datasrc.Error. If that happens, you should stop
using this object and abort the modification.
"""
self.__check_commited()
# First, compact the data
self.compact()
try:
# Then pass the data inside the data source
for (operation, rrset) in self.__buffer:
if operation == 'add':
self.__updater.add_rrset(rrset)
elif operation == 'remove':
self.__updater.remove_rrset(rrset)
else:
raise ValueError('Unknown operation ' + operation)
# As everything is already in, drop the buffer
except:
# If there's a problem, we can't continue.
self.__updater = None
raise
self.__buffer = []
def commit(self):
"""
Writes all the changes into the data source and makes them visible.
This closes the diff, you may not use it any more. If you try to use
it, you'll get ValueError.
This might raise isc.datasrc.Error.
"""
self.__check_commited()
# Push the data inside the data source
self.apply()
# Make sure they are visible.
try:
self.__updater.commit()
finally:
# Remove the updater. That will free some resources for one, but
# mark this object as already commited, so we can check
# We remove it even in case the commit failed, as that makes us
# unusable.
self.__updater = None
def get_buffer(self):
"""
Returns the current buffer of changes not yet passed into the data
source. It is in a form like [('add', rrset), ('remove', rrset),
('remove', rrset), ...].
Probably useful only for testing and introspection purposes. Don't
modify the list.
"""
return self.__buffer
# Copyright (C) 2011 Internet Systems Consortium, Inc. ("ISC")
#
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
# AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
# PERFORMANCE OF THIS SOFTWARE.
# No namespace declaration - these constants go in the global namespace
# of the libxfrin_messages python module.
% LIBXFRIN_DIFFERENT_TTL multiple data with different TTLs (%1, %2) on %3/%4. Adjusting %2 -> %1.
The xfrin module received an update containing multiple rdata changes for the
same RRset. But the TTLs of these don't match each other. As we combine them
together, the later one get's overwritten to the earlier one in the sequence.
PYCOVERAGE_RUN = @PYCOVERAGE_RUN@
PYTESTS = diff_tests.py
EXTRA_DIST = $(PYTESTS)
# If necessary (rare cases), explicitly specify paths to dynamic libraries
# required by loadable python modules.
LIBRARY_PATH_PLACEHOLDER =
if SET_ENV_LIBRARY_PATH
LIBRARY_PATH_PLACEHOLDER += $(ENV_LIBRARY_PATH)=$(abs_top_builddir)/src/lib/cryptolink/.libs:$(abs_top_builddir)/src/lib/dns/.libs:$(abs_top_builddir)/src/lib/dns/python/.libs:$(abs_top_builddir)/src/lib/cc/.libs:$(abs_top_builddir)/src/lib/config/.libs:$(abs_top_builddir)/src/lib/log/.libs:$(abs_top_builddir)/src/lib/util/.libs:$(abs_top_builddir)/src/lib/exceptions/.libs:$(abs_top_builddir)/src/lib/datasrc/.libs:$$$(ENV_LIBRARY_PATH)
endif
# test using command-line arguments, so use check-local target instead of TESTS
check-local:
if ENABLE_PYTHON_COVERAGE
touch $(abs_top_srcdir)/.coverage
rm -f .coverage
${LN_S} $(abs_top_srcdir)/.coverage .coverage
endif
for pytest in $(PYTESTS) ; do \
echo Running test: $$pytest ; \
$(LIBRARY_PATH_PLACEHOLDER) \
PYTHONPATH=$(COMMON_PYTHON_PATH):$(abs_top_builddir)/src/lib/dns/python/.libs \
$(PYCOVERAGE_RUN) $(abs_srcdir)/$$pytest || exit ; \
done
# Copyright (C) 2011 Internet Systems Consortium.
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM
# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT,
# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
import isc.log
import unittest
from isc.dns import Name, RRset, RRClass, RRType, RRTTL, Rdata
from isc.xfrin.diff import Diff, NoSuchZone
class TestError(Exception):
"""
Just to have something to be raised during the tests.
Not used outside.
"""
pass
class DiffTest(unittest.TestCase):
"""
Tests for the isc.xfrin.diff.Diff class.
It also plays role of a data source and an updater, so it can manipulate
some test variables while being called.
"""
def setUp(self):
"""
This sets internal variables so we can see nothing was called yet.
It also creates some variables used in multiple tests.
"""
# Track what was called already
self.__updater_requested = False
self.__compact_called = False
self.__data_operations = []
self.__apply_called = False
self.__commit_called = False
self.__broken_called = False
self.__warn_called = False
# Some common values
self.__rrclass = RRClass.IN()
self.__type = RRType.A()
self.__ttl = RRTTL(3600)
# And RRsets
# Create two valid rrsets
self.__rrset1 = RRset(Name('a.example.org.'), self.__rrclass,
self.__type, self.__ttl)
self.__rdata = Rdata(self.__type, self.__rrclass, '192.0.2.1')
self.__rrset1.add_rdata(self.__rdata)
self.__rrset2 = RRset(Name('b.example.org.'), self.__rrclass,
self.__type, self.__ttl)
self.__rrset2.add_rdata(self.__rdata)
# And two invalid
self.__rrset_empty = RRset(Name('empty.example.org.'), self.__rrclass,
self.__type, self.__ttl)
self.__rrset_multi = RRset(Name('multi.example.org.'), self.__rrclass,
self.__type, self.__ttl)
self.__rrset_multi.add_rdata(self.__rdata)
self.__rrset_multi.add_rdata(Rdata(self.__type, self.__rrclass,
'192.0.2.2'))
def __mock_compact(self):
"""
This can be put into the diff to hook into its compact method and see
if it gets called.
"""
self.__compact_called = True
def __mock_apply(self):
"""
This can be put into the diff to hook into its apply method and see
it gets called.
"""
self.__apply_called = True
def __broken_operation(self, *args):
"""
This can be used whenever an operation should fail. It raises TestError.
It should take whatever amount of parameters needed, so it can be put
quite anywhere.
"""
self.__broken_called = True
raise TestError("Test error")
def warn(self, *args):
"""
This is for checking the warn function was called, we replace the logger
in the tested module.
"""
self.__warn_called = True
def commit(self):
"""
This is part of pretending to be a zone updater. This notes the commit
was called.
"""
self.__commit_called = True
def add_rrset(self, rrset):
"""
This one is part of pretending to be a zone updater. It writes down
addition of an rrset was requested.
"""
self.__data_operations.append(('add', rrset))
def remove_rrset(self, rrset):
"""
This one is part of pretending to be a zone updater. It writes down
removal of an rrset was requested.
"""
self.__data_operations.append(('remove', rrset))
def get_class(self):
"""
This one is part of pretending to be a zone updater. It returns
the IN class.
"""
return self.__rrclass
def get_updater(self, zone_name, replace):
"""
This one pretends this is the data source client and serves
getting an updater.
If zone_name is 'none.example.org.', it returns None, otherwise
it returns self.
"""
# The diff should not delete the old data.
self.assertFalse(replace)
self.__updater_requested = True
# Pretend this zone doesn't exist
if zone_name == Name('none.example.org.'):
return None
else:
return self
def test_create(self):
"""
This test the case when the diff is successfuly created. It just
tries it does not throw and gets the updater.
"""
diff = Diff(self, Name('example.org.'))
self.assertTrue(self.__updater_requested)
self.assertEqual([], diff.get_buffer())
def test_create_nonexist(self):
"""
Try to create a diff on a zone that doesn't exist. This should
raise a correct exception.
"""
self.assertRaises(NoSuchZone, Diff, self, Name('none.example.org.'))
self.assertTrue(self.__updater_requested)
def __data_common(self, diff, method, operation):
"""
Common part of test for test_add and test_remove.
"""
# Try putting there the bad data first
self.assertRaises(ValueError, method, self.__rrset_empty)
self.assertRaises(ValueError, method, self.__rrset_multi)
# They were not added
self.assertEqual([], diff.get_buffer())
# Put some proper data into the diff
method(self.__rrset1)
method(self.__rrset2)
dlist = [(operation, self.__rrset1), (operation, self.__rrset2)]
self.assertEqual(dlist, diff.get_buffer())
# Check the data are not destroyed by raising an exception because of
# bad data
self.assertRaises(ValueError, method, self.__rrset_empty)
self.assertEqual(dlist, diff.get_buffer())
def test_add(self):
"""
Try to add few items into the diff and see they are stored in there.
Also try passing an rrset that has differnt amount of RRs than 1.
"""
diff = Diff(self, Name('example.org.'))
self.__data_common(diff, diff.add_data, 'add')
def test_remove(self):
"""
Try scheduling removal of few items into the diff and see they are
stored in there.
Also try passing an rrset that has different amount of RRs than 1.
"""
diff = Diff(self, Name('example.org.'))
self.__data_common(diff, diff.remove_data, 'remove')
def test_apply(self):
"""
Schedule few additions and check the apply works by passing the
data into the updater.
"""
# Prepare the diff
diff = Diff(self, Name('example.org.'))
diff.add_data(self.__rrset1)
diff.remove_data(self.__rrset2)
dlist = [('add', self.__rrset1), ('remove', self.__rrset2)]
self.assertEqual(dlist, diff.get_buffer())
# Do the apply, hook the compact method
diff.compact = self.__mock_compact
diff.apply()
# It should call the compact
self.assertTrue(self.__compact_called)
# And pass the data. Our local history of what happened is the same
# format, so we can check the same way
self.assertEqual(dlist, self.__data_operations)
# And the buffer in diff should become empty, as everything
# got inside.
self.assertEqual([], diff.get_buffer())
def test_commit(self):
"""
If we call a commit, it should first apply whatever changes are
left (we hook into that instead of checking the effect) and then
the commit on the updater should have been called.
Then we check it raises value error for whatever operation we try.
"""
diff = Diff(self, Name('example.org.'))
diff.add_data(self.__rrset1)
orig_apply = diff.apply
diff.apply = self.__mock_apply
diff.commit()
self.assertTrue(self.__apply_called)
self.assertTrue(self.__commit_called)
# The data should be handled by apply which we replaced.
self.assertEqual([], self.__data_operations)
# Now check all range of other methods raise ValueError
self.assertRaises(ValueError, diff.commit)
self.assertRaises(ValueError, diff.add_data, self.__rrset2)
self.assertRaises(ValueError, diff.remove_data, self.__rrset1)
diff.apply = orig_apply
self.assertRaises(ValueError, diff.apply)
# This one does not state it should raise, so check it doesn't
# But it is NOP in this situation anyway
diff.compact()
def test_autoapply(self):
"""
Test the apply is called all by itself after 100 tasks are added.
"""
diff = Diff(self, Name('example.org.'))
# A method to check the apply is called _after_ the 100th element
# is added. We don't use it anywhere else, so we define it locally
# as lambda function
def check():
self.assertEqual(100, len(diff.get_buffer()))
self.__mock_apply()
orig_apply = diff.apply
diff.apply = check
# If we put 99, nothing happens yet
for i in range(0, 99):
diff.add_data(self.__rrset1)
expected = [('add', self.__rrset1)] * 99