Commit 72fb34ad authored by JINMEI Tatuya's avatar JINMEI Tatuya
Browse files

[2380] Merge branch 'trac2379' into trac2380

parents 6260781b 52b93e86
......@@ -777,6 +777,10 @@ PyInit_pydnspp(void) {
po_IscException = PyErr_NewException("pydnspp.IscException", NULL, NULL);
PyModule_AddObject(mod, "IscException", po_IscException);
po_InvalidOperation = PyErr_NewException("pydnspp.InvalidOperation",
NULL, NULL);
PyModule_AddObject(mod, "InvalidOperation", po_InvalidOperation);
po_InvalidParameter = PyErr_NewException("pydnspp.InvalidParameter",
NULL, NULL);
PyModule_AddObject(mod, "InvalidParameter", po_InvalidParameter);
......
......@@ -47,6 +47,7 @@ namespace dns {
namespace python {
// For our 'general' isc::Exceptions
PyObject* po_IscException;
PyObject* po_InvalidOperation;
PyObject* po_InvalidParameter;
// For our own isc::dns::Exception
......
......@@ -28,6 +28,7 @@ namespace dns {
namespace python {
// For our 'general' isc::Exceptions
extern PyObject* po_IscException;
extern PyObject* po_InvalidOperation;
extern PyObject* po_InvalidParameter;
// For our own isc::dns::Exception
......
......@@ -20,6 +20,7 @@ datasrc_la_SOURCES += updater_python.cc updater_python.h
datasrc_la_SOURCES += journal_reader_python.cc journal_reader_python.h
datasrc_la_SOURCES += configurableclientlist_python.cc
datasrc_la_SOURCES += configurableclientlist_python.h
datasrc_la_SOURCES += zone_loader_python.cc zone_loader_python.h
datasrc_la_CPPFLAGS = $(AM_CPPFLAGS) $(PYTHON_INCLUDES)
datasrc_la_CXXFLAGS = $(AM_CXXFLAGS) $(PYTHON_CXXFLAGS)
......@@ -35,6 +36,7 @@ EXTRA_DIST += finder_inc.cc
EXTRA_DIST += iterator_inc.cc
EXTRA_DIST += updater_inc.cc
EXTRA_DIST += journal_reader_inc.cc
EXTRA_DIST += zone_loader_inc.cc
CLEANDIRS = __pycache__
......
......@@ -2,7 +2,7 @@ import sys
import os
# The datasource factory loader uses dlopen, as does python
# for its modules. Some dynamic linkers do not play nice if
# for its modules. Some dynamic linkers do not play nice if
# modules are not loaded with RTLD_GLOBAL, a symptom of which
# is that exceptions are not recognized by type. So to make
# sure this doesn't happen, we temporarily set RTLD_GLOBAL
......@@ -31,5 +31,4 @@ else:
sys.setdlopenflags(flags)
from isc.datasrc.sqlite3_ds import *
from isc.datasrc.master import *
......@@ -374,6 +374,17 @@ wrapDataSourceClient(DataSourceClient* client,
return (container.release());
}
DataSourceClient&
PyDataSourceClient_ToDataSourceClient(PyObject* client_obj) {
if (client_obj == NULL) {
isc_throw(PyCPPWrapperException,
"obj argument NULL in Name PyObject conversion");
}
const s_DataSourceClient* client =
static_cast<const s_DataSourceClient*>(client_obj);
return (*client->client);
}
} // namespace python
} // namespace datasrc
} // namespace isc
......@@ -44,6 +44,9 @@ wrapDataSourceClient(DataSourceClient* client,
LifeKeeper>& life_keeper = boost::shared_ptr<ClientList::
FindResult::LifeKeeper>());
DataSourceClient&
PyDataSourceClient_ToDataSourceClient(PyObject* client_obj);
} // namespace python
} // namespace datasrc
} // namespace isc
......
......@@ -29,6 +29,7 @@
#include "updater_python.h"
#include "journal_reader_python.h"
#include "configurableclientlist_python.h"
#include "zone_loader_python.h"
#include <util/python/pycppwrapper_util.h>
#include <dns/python/pydnspp_common.h>
......@@ -180,6 +181,23 @@ initModulePart_ZoneIterator(PyObject* mod) {
return (true);
}
bool
initModulePart_ZoneLoader(PyObject* mod) {
// We initialize the static description object with PyType_Ready(),
// then add it to the module. This is not just a check! (leaving
// this out results in segmentation faults)
if (PyType_Ready(&zone_loader_type) < 0) {
return (false);
}
void* p = &zone_loader_type;
if (PyModule_AddObject(mod, "ZoneLoader", static_cast<PyObject*>(p)) < 0) {
return (false);
}
Py_INCREF(&zone_loader_type);
return (true);
}
bool
initModulePart_ZoneUpdater(PyObject* mod) {
// We initialize the static description object with PyType_Ready(),
......@@ -234,8 +252,9 @@ initModulePart_ZoneJournalReader(PyObject* mod) {
}
PyObject* po_DataSourceError;
PyObject* po_OutOfZone;
PyObject* po_MasterFileError;
PyObject* po_NotImplemented;
PyObject* po_OutOfZone;
PyModuleDef iscDataSrc = {
{ PyObject_HEAD_INIT(NULL) NULL, 0, NULL},
......@@ -260,6 +279,26 @@ PyInit_datasrc(void) {
return (NULL);
}
try {
po_DataSourceError = PyErr_NewException("isc.datasrc.Error", NULL,
NULL);
PyObjectContainer(po_DataSourceError).installToModule(mod, "Error");
po_MasterFileError = PyErr_NewException("isc.datasrc.MasterFileError",
po_DataSourceError, NULL);
PyObjectContainer(po_MasterFileError).
installToModule(mod, "MasterFileError");
po_OutOfZone = PyErr_NewException("isc.datasrc.OutOfZone", NULL, NULL);
PyObjectContainer(po_OutOfZone).installToModule(mod, "OutOfZone");
po_NotImplemented = PyErr_NewException("isc.datasrc.NotImplemented",
NULL, NULL);
PyObjectContainer(po_NotImplemented).installToModule(mod,
"NotImplemented");
} catch (...) {
Py_DECREF(mod);
return (NULL);
}
if (!initModulePart_DataSourceClient(mod)) {
Py_DECREF(mod);
return (NULL);
......@@ -290,17 +329,7 @@ PyInit_datasrc(void) {
return (NULL);
}
try {
po_DataSourceError = PyErr_NewException("isc.datasrc.Error", NULL,
NULL);
PyObjectContainer(po_DataSourceError).installToModule(mod, "Error");
po_OutOfZone = PyErr_NewException("isc.datasrc.OutOfZone", NULL, NULL);
PyObjectContainer(po_OutOfZone).installToModule(mod, "OutOfZone");
po_NotImplemented = PyErr_NewException("isc.datasrc.NotImplemented",
NULL, NULL);
PyObjectContainer(po_NotImplemented).installToModule(mod,
"NotImplemented");
} catch (...) {
if (!initModulePart_ZoneLoader(mod)) {
Py_DECREF(mod);
return (NULL);
}
......
......@@ -182,7 +182,7 @@ def records(input):
if paren == 1 or not record:
continue
ret = ' '.join(record)
record = []
oldsize = size
......
PYCOVERAGE_RUN = @PYCOVERAGE_RUN@
# old tests, TODO remove or change to use new API?
#PYTESTS = master_test.py
PYTESTS = datasrc_test.py sqlite3_ds_test.py clientlist_test.py
PYTESTS = datasrc_test.py sqlite3_ds_test.py
PYTESTS += clientlist_test.py zone_loader_test.py
EXTRA_DIST = $(PYTESTS)
EXTRA_DIST += testdata/brokendb.sqlite3
......@@ -9,7 +10,10 @@ EXTRA_DIST += testdata/example.com.sqlite3
EXTRA_DIST += testdata/newschema.sqlite3
EXTRA_DIST += testdata/oldschema.sqlite3
EXTRA_DIST += testdata/new_minor_schema.sqlite3
EXTRA_DIST += testdata/example.com
EXTRA_DIST += testdata/example.com.ch
CLEANFILES = $(abs_builddir)/rwtest.sqlite3.copied
CLEANFILES += $(abs_builddir)/zoneloadertest.sqlite3
# If necessary (rare cases), explicitly specify paths to dynamic libraries
# required by loadable python modules.
......
example.com. 1000 IN SOA a.dns.example.com. mail.example.com. 1 1 1 1 1
example.com. 1000 IN NS a.dns.example.com.
example.com. 1000 IN NS b.dns.example.com.
example.com. 1000 IN NS c.dns.example.com.
a.dns.example.com. 1000 IN A 1.1.1.1
b.dns.example.com. 1000 IN A 3.3.3.3
b.dns.example.com. 1000 IN AAAA 4:4::4:4
b.dns.example.com. 1000 IN AAAA 5:5::5:5
example.com. 1000 CH SOA a.dns.example.com. mail.example.com. 1 1 1 1 1
example.com. 1000 CH NS a.dns.example.com.
example.com. 1000 CH NS b.dns.example.com.
example.com. 1000 CH NS c.dns.example.com.
a.dns.example.com. 1000 CH A 1.1.1.1
b.dns.example.com. 1000 CH A 3.3.3.3
b.dns.example.com. 1000 CH AAAA 4:4::4:4
b.dns.example.com. 1000 CH AAAA 5:5::5:5
# Copyright (C) 2012 Internet Systems Consortium.
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM
# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT,
# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
import isc.datasrc
import isc.dns
import os
import unittest
import shutil
# Constants and common data used in tests
TESTDATA_PATH = os.environ['TESTDATA_PATH']
TESTDATA_WRITE_PATH = os.environ['TESTDATA_WRITE_PATH']
ZONE_FILE = TESTDATA_PATH + '/example.com'
STATIC_ZONE_FILE = '../../../../datasrc/static.zone'
SOURCE_DB_FILE = TESTDATA_PATH + '/example.com.source.sqlite3'
ORIG_DB_FILE = TESTDATA_PATH + '/example.com.sqlite3'
DB_FILE = TESTDATA_WRITE_PATH + '/zoneloadertest.sqlite3'
DB_CLIENT_CONFIG = '{ "database_file": "' + DB_FILE + '" }'
DB_SOURCE_CLIENT_CONFIG = '{ "database_file": "' + SOURCE_DB_FILE + '" }'
ORIG_SOA_TXT = 'example.com. 3600 IN SOA master.example.com. ' +\
'admin.example.com. 1234 3600 1800 2419200 7200\n'
NEW_SOA_TXT = 'example.com. 1000 IN SOA a.dns.example.com. ' +\
'mail.example.com. 1 1 1 1 1\n'
class ZoneLoaderTests(unittest.TestCase):
def setUp(self):
self.test_name = isc.dns.Name("example.com")
self.test_file = ZONE_FILE
self.client = isc.datasrc.DataSourceClient("sqlite3", DB_CLIENT_CONFIG)
# Make a fresh copy of the database
shutil.copy(ORIG_DB_FILE, DB_FILE)
def test_bad_constructor(self):
self.assertRaises(TypeError, isc.datasrc.ZoneLoader)
self.assertRaises(TypeError, isc.datasrc.ZoneLoader, 1)
self.assertRaises(TypeError, isc.datasrc.ZoneLoader,
None, self.test_name, self.test_file)
self.assertRaises(TypeError, isc.datasrc.ZoneLoader,
self.client, None, self.test_file)
self.assertRaises(TypeError, isc.datasrc.ZoneLoader,
self.client, self.test_name, None)
self.assertRaises(TypeError, isc.datasrc.ZoneLoader,
self.client, self.test_name, self.test_file, 1)
def check_zone_soa(self, soa_txt):
"""
Check that the given SOA RR exists and matches the expected string
"""
result, finder = self.client.find_zone(self.test_name)
self.assertEqual(self.client.SUCCESS, result)
result, rrset, _ = finder.find(self.test_name, isc.dns.RRType.SOA())
self.assertEqual(finder.SUCCESS, result)
self.assertEqual(soa_txt, rrset.to_text())
def check_load(self, loader):
self.check_zone_soa(ORIG_SOA_TXT)
loader.load()
self.check_zone_soa(NEW_SOA_TXT)
# And after that, it should throw
self.assertRaises(isc.dns.InvalidOperation, loader.load)
def test_load_from_file(self):
loader = isc.datasrc.ZoneLoader(self.client, self.test_name,
self.test_file)
self.check_load(loader)
def test_load_from_client(self):
source_client = isc.datasrc.DataSourceClient('sqlite3',
DB_SOURCE_CLIENT_CONFIG)
loader = isc.datasrc.ZoneLoader(self.client, self.test_name,
source_client)
self.check_load(loader)
def check_load_incremental(self, loader):
# New zone has 8 RRs
# After 5, it should return False
self.assertFalse(loader.load_incremental(5))
# New zone should not have been loaded yet
self.check_zone_soa(ORIG_SOA_TXT)
# After 5 more, it should return True (only having read 3)
self.assertTrue(loader.load_incremental(5))
# New zone should now be loaded
self.check_zone_soa(NEW_SOA_TXT)
# And after that, it should throw
self.assertRaises(isc.dns.InvalidOperation, loader.load_incremental, 5)
def test_load_from_file_incremental(self):
# Create loader and load the zone
loader = isc.datasrc.ZoneLoader(self.client, self.test_name,
self.test_file)
self.check_load_incremental(loader)
def test_load_from_client_incremental(self):
source_client = isc.datasrc.DataSourceClient('sqlite3',
DB_SOURCE_CLIENT_CONFIG)
loader = isc.datasrc.ZoneLoader(self.client, self.test_name,
source_client)
self.check_load_incremental(loader)
def test_bad_file(self):
self.check_zone_soa(ORIG_SOA_TXT)
loader = isc.datasrc.ZoneLoader(self.client, self.test_name,
'no such file')
self.assertRaises(isc.datasrc.MasterFileError, loader.load)
self.check_zone_soa(ORIG_SOA_TXT)
def test_bad_file_incremental(self):
self.check_zone_soa(ORIG_SOA_TXT)
loader = isc.datasrc.ZoneLoader(self.client, self.test_name,
'no such file')
self.assertRaises(isc.datasrc.MasterFileError,
loader.load_incremental, 1)
self.check_zone_soa(ORIG_SOA_TXT)
def test_no_such_zone_in_target(self):
self.assertRaises(isc.datasrc.Error, isc.datasrc.ZoneLoader,
self.client, isc.dns.Name("unknownzone"),
self.test_file)
def test_no_such_zone_in_source(self):
source_client = isc.datasrc.DataSourceClient('sqlite3',
DB_SOURCE_CLIENT_CONFIG)
self.assertRaises(isc.datasrc.Error, isc.datasrc.ZoneLoader,
self.client, isc.dns.Name("unknownzone"),
source_client)
def test_no_ds_load_support(self):
# This may change in the future, but atm, the in-mem ds does
# not support the API the zone loader uses (it has direct load calls)
inmem_client = isc.datasrc.DataSourceClient('memory',
'{ "type": "memory" }');
self.assertRaises(isc.datasrc.NotImplemented,
isc.datasrc.ZoneLoader,
inmem_client, self.test_name, self.test_file)
def test_wrong_class_from_file(self):
# If the file has wrong class, it is not detected until load time
loader = isc.datasrc.ZoneLoader(self.client, self.test_name,
self.test_file + '.ch')
self.assertRaises(isc.datasrc.MasterFileError, loader.load)
def test_wrong_class_from_client(self):
# For ds->ds loading, wrong class is detected upon construction
# Need a bit of the extended setup for CH source client
clientlist = isc.datasrc.ConfigurableClientList(isc.dns.RRClass.CH())
clientlist.configure('[ { "type": "static", "params": "' +
STATIC_ZONE_FILE +'" } ]', False)
source_client, _, _ = clientlist.find(isc.dns.Name("bind."),
False, False)
self.assertRaises(isc.dns.InvalidParameter, isc.datasrc.ZoneLoader,
self.client, isc.dns.Name("bind."), source_client)
def test_exception(self):
# Just check if masterfileerror is subclass of datasrc.Error
self.assertTrue(issubclass(isc.datasrc.MasterFileError,
isc.datasrc.Error))
if __name__ == "__main__":
isc.log.init("bind10")
isc.log.resetUnitTestRootLogger()
unittest.main()
namespace {
const char* const ZoneLoader_doc = "\
\n\
Class to load data into a data source client.\n\
\n\
This is a small wrapper class that is able to load data into a data source.\n\
It can load either from another data source or from a master file. The\n\
purpose of the class is only to hold the state for incremental loading.\n\
\n\
The old content of zone is discarded and no journal is stored.\n\
\n\
The constructor takes three arguments:\n\
- The datasource (isc.datasrc.DataSourceClient) to load the zone into\n\
- The name (isc.dns.Name) to load\n\
- either a string (for a file) or another DataSourceClient to load from\n\
\n\
Upon construction, no loading is done yet.\n\
\n\
It can throw:\n\
DataSourceError, in case the zone does not exist in destination.\n\
This class does not support creating brand new zones, only loading\n\
data into them. In case a new zone is needed, it must be created\n\
beforehand (with create_zone()).\n\
DataSourceError is also thrown in case the zone is not present in the\n\
source DataSourceClient, and in case of other possibly low-level\n\
errors.\n\
InvalidParameter, in case the class of destination and source\n\
differs.\n\
NotImplemented in case target data source client doesn't provide an updater\n\
or the source data source client doesn't provide an iterator.\n\
\n\
";
const char* const ZoneLoader_loadIncremental_doc = "\
\n\
Load up to limit RRs.\n\
\n\
This performs a part of the loading. In case there's enough data in the\n\
source, it copies limit RRs. It can copy less RRs during the final call\n\
(when there's less than limit left).\n\
\n\
This can be called repeatedly until the whole zone is loaded, having\n\
pauses in the loading for some purposes (for example reporting\n\
progress).\n\
\n\
It has one parameter: limit (integer), The maximum allowed number of RRs\n\
to be loaded during this call.\n\
\n\
Returns True in case the loading is completed, and False if there's more\n\
to load.\n\
\n\
It can throw:\n\
InvalidOperation, in case the loading was already completed before this\n\
call (by load() or by a loadIncremental that returned true).\n\
DataSourceError, in case some error (possibly low-level) happens.\n\
MasterFileError when the master_file is badly formatted or some similar\n\
problem is found when loading the master file.\n\
\n\
Note: If the limit is exactly the number of RRs available to be loaded,\n\
the method still returns false and true'll be returned on the next\n\
call (which will load 0 RRs). This is because the end of iterator or\n\
master file is detected when reading past the end, not when the last\n\
one is read.\n\
\n\
";
const char* const ZoneLoader_load_doc = "\
\n\
Performs the entire load operation.\n\
\n\
Depending on zone size, this could take a long time.\n\
\n\
This method has no parameters and does not return anything.\n\
\n\
It can throw:\n\
InvalidOperation, in case the loading was already completed before this call.\n\
MasterFileError, when the master_file is badly formatted or some\n\
similar problem is found when loading the master file.\n\
DataSourceError, in case some error (possibly low-level) happens.\n\
\n\
";
} // unnamed namespace
// Copyright (C) 2012 Internet Systems Consortium, Inc. ("ISC")
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
// Enable this if you use s# variants with PyArg_ParseTuple(), see
// http://docs.python.org/py3k/c-api/arg.html#strings-and-buffers
//#define PY_SSIZE_T_CLEAN
// Python.h needs to be placed at the head of the program file, see:
// http://docs.python.org/py3k/extending/extending.html#a-simple-example
#include <Python.h>
#include <util/python/pycppwrapper_util.h>
#include <datasrc/zone_loader.h>
#include <dns/python/name_python.h>
#include <dns/python/pydnspp_common.h>
#include <exceptions/exceptions.h>
#include "client_python.h"
#include "datasrc.h"
#include "zone_loader_inc.cc"
using namespace std;
using namespace isc::dns::python;
using namespace isc::datasrc;
using namespace isc::datasrc::python;
namespace {
// The s_* Class simply covers one instantiation of the object
class s_ZoneLoader : public PyObject {
public:
s_ZoneLoader() : cppobj(NULL), client(NULL) {};
ZoneLoader* cppobj;
// a zoneloader should not survive its associated client,
// so add a ref to it at init
PyObject* client;
};
// General creation and destruction
int
ZoneLoader_init(PyObject* po_self, PyObject* args, PyObject*) {
s_ZoneLoader* self = static_cast<s_ZoneLoader*>(po_self);
PyObject *po_target_client = NULL;
PyObject *po_source_client = NULL;
PyObject *po_name = NULL;
char* master_file;
if (!PyArg_ParseTuple(args, "O!O!s", &datasourceclient_type,
&po_target_client, &name_type, &po_name,
&master_file) &&
!PyArg_ParseTuple(args, "O!O!O!", &datasourceclient_type,
&po_target_client, &name_type, &po_name,
&datasourceclient_type, &po_source_client)
) {
return (-1);
}
PyErr_Clear();
try {
Py_INCREF(po_target_client);
self->client = po_target_client;
if (po_source_client != NULL) {
self->cppobj = new ZoneLoader(
PyDataSourceClient_ToDataSourceClient(po_target_client),
PyName_ToName(po_name),
PyDataSourceClient_ToDataSourceClient(po_source_client));
} else {
self->cppobj = new ZoneLoader(
PyDataSourceClient_ToDataSourceClient(po_target_client),
PyName_ToName(po_name),
master_file);
}
return (0);
} catch (const isc::InvalidParameter& ivp) {
PyErr_SetString(po_InvalidParameter, ivp.what());
} catch (const isc::datasrc::DataSourceError& dse) {
PyErr_SetString(getDataSourceException("Error"), dse.what());
} catch (const isc::NotImplemented& ni) {
PyErr_SetString(getDataSourceException("NotImplemented"), ni.what());
} catch (const std::exception& stde) {
PyErr_SetString(getDataSourceException("Error"), stde.what());
} catch (...) {
PyErr_SetString(getDataSourceException("Error"),
"Unexpected exception");
}
return (-1);
}
void
ZoneLoader_destroy(PyObject* po_self) {
s_ZoneLoader* self = static_cast<s_ZoneLoader*>(po_self