Commit 55b3152e authored by JINMEI Tatuya's avatar JINMEI Tatuya
Browse files

[trac905] convenient utilities to write python C++ bindings more safely and easily.

parent e93f054a
#!@PYTHON@
# Copyright (C) 2011 Internet Systems Consortium.
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM
# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT,
# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
"""This utility program generates a C++ header and implementation files
that can be used as a template of C++ python binding for a C++ class.
Usage: ./mkpywrapper.py ClassName
(the script should be run on this directory)
It will generate two files: classname_python.h and classname_python.cc,
many of whose definitions are in the namespace isc::MODULE_NAME::python.
By default MODULE_NAME will be 'dns' (because this tool is originally
intended to be used for the C++ python binding of the DNS library), but
can be changed via the -m command line option.
The generated files contain code fragments that are commonly used in
C++ python binding implementations. It will define a class named
s_ClassName which is a derived class of PyModule and can meet the
requirement of the CPPPyObjectContainer template class (see
pycppwrapper_util.h). It also defines (and declares in the header file)
"classname_type", which is of PyTypeObject and is intended to be used
to define details of the python bindings for the ClassName class.
In many cases the header file can be used as a startpoint of the
binding development without modification. But you may want to make
ClassName::cppobj a constant variable (and you should if you can).
Many definitions of classname_python.cc should also be able to be used
just as defined, but some will need to be changed or removed. In
particular, you should at least adjust ClassName_init(). You'll
probably also need to add more definitions to that file to provide
complete features of the C++ class.
"""
import datetime, string, sys
from optparse import OptionParser
# Remember the current year to produce the copyright boilerplate
YEAR = datetime.date.today().timetuple()[0]
def dump_file(out_file, temp_file, class_name, module):
for line in temp_file.readlines():
line = line.replace("@YEAR@", str(YEAR))
line = line.replace("@CPPCLASS@_H", class_name.upper() + "_H")
line = line.replace("@CPPCLASS@", class_name)
line = line.replace("@cppclass@", class_name.lower())
line = line.replace("@MODULE@", module)
out_file.write(line)
def dump_wrappers(class_name, output, module):
try:
if output == "-":
header_file = sys.stdout
else:
header_file = open(output + "_python.h", "w")
header_template_file = open("wrapper_template.h", "r")
if output == "-":
impl_file = sys.stdout
else:
impl_file = open(class_name.lower() + "_python.cc", "w")
impl_template_file = open("wrapper_template.cc", "r")
except:
sys.stderr.write('Failed to open C++ file(s)\n')
sys.exit(1)
dump_file(header_file, header_template_file, class_name, module)
dump_file(impl_file, impl_template_file, class_name, module)
usage = '''usage: %prog [options] class_name'''
if __name__ == "__main__":
parser = OptionParser(usage=usage)
parser.add_option('-o', '--output', action='store', dest='output',
default=None, metavar='FILE',
help='prefix of output file names [default: derived from the class name]')
parser.add_option('-m', '--module', action='store', dest='module',
default='dns',
help='C++ module name of the wrapper (for namespaces) [default: dns]')
(options, args) = parser.parse_args()
if len(args) == 0:
parser.error('input file is missing')
class_name = args[0]
if not options.output:
options.output = class_name.lower()
dump_wrappers(class_name, options.output, options.module)
// Copyright (C) 2011 Internet Systems Consortium, Inc. ("ISC")
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
#ifndef __PYCPPWRAPPER_UTIL_H
#define __PYCPPWRAPPER_UTIL_H 1
#include <Python.h>
#include <exceptions/exceptions.h>
/**
* @file pycppwrapper_util.h
* @short Shared definitions for python/C(++) API
*
* This utility defines a set of convenient wrappers for the python C API
* to use it safely from our C++ bindings. The python C API has many pitfalls
* such as not-so-consistent reference count policies. Also, many existing
* examples are careless about error handling. It's easy to find on the net
* example (even of "production use") python extensions like this:
*
* \code
* new_exception = PyErr_NewException("mymodule.Exception", NULL, NULL);
* // new_exception can be NULL, in which case the call to
* // PyModule_AddObject will cause a surprising disruption.
* PyModule_AddObject(mymodule, "Exception", new_exception); \endcode
*
* When using the python C API with C++, we should also be careful about
* exception safety. The underlying C++ code (including standard C++ libraries
* and memory allocation) can throw exceptions, in which case we need to
* make sure any intermediate python objects are cleaned up (we also need to
* catch the C++ exceptions inside the binding and convert them to python
* errors, but that's a different subject). This is not a trivial task
* because the python objects are represented as bare C pointers (so there's
* no destructor) and we need to address the exception safety along with python
* reference counters (so we cannot naively apply standard smart pointers).
*
* This utility tries to help address these issues.
*
* Also, it's intentional that this is a header-only utility. This way the
* C++ loadable module won't depend on another C++ library (which is not
* necessarily wrong, but would increase management cost such as link-time
* troubles only for a small utility feature).
*/
namespace isc {
namespace util {
namespace python {
/// This is thrown inside this utility when it finds a NULL pointer is passed
/// when it should not be NULL.
class PyCPPWrapperException : public isc::Exception {
public:
PyCPPWrapperException(const char* file, size_t line, const char* what) :
isc::Exception(file, line, what) {}
};
/// This helper class is similar to the standard autoptr and manages PyObject
/// using some kind of RAII techniques. It is, however, customized for the
/// the python C API.
///
/// A PyObjectContainer object is constructed with a pointer to PyObject,
/// which is often just created dynamically. The caller will eventually
/// attach the object to a different python object (often a module or class)
/// via specific methods or directly return it to the python interpreter.
///
/// There are two cases in destructing the object: with or without decreasing
/// a reference to the PyObject. If the object is intended to be an argument
/// to another python C library that increases the reference to the object for
/// itself, we should normally release our own reference; otherwise the
/// reference will leak and the object won't be garbage collected. Also, when
/// an unexpected error happens in the form of C++ exception, we should
/// release the reference to prevent resource leak.
///
/// In some other cases, we should simply give our reference to the caller.
/// That is the case when the created object itself is a return value of
/// an extended python method written in the C++ binding. Likewise, some
/// python C library functions "steal" the reference. In these cases we
/// should not decrease the reference; otherwise it would cause duplicate free.
///
/// By default, the destructor of this class releases the reference to the
/// PyObject. If this behavior is desirable, you can extract the original
/// bare pointer to the PyObject by the \c get() method. If you don't want
/// the reference to be decrease, the original bare pointer should be
/// extracted using the \c release() method.
///
/// There are two convenience methods for commonly used operations:
/// \c installAsClassVariable() to add the PyObject as a class variable
/// and \c installToModule to add the PyObject to a specified python module.
/// These methods (at least to some extent) take care of the reference to
/// the object (either release or keep) depending on the usage context so
/// that the user don't have to worry about it.
///
/// On construction, this class expects the pointer can be NULL.
/// If it happens it immediately throws a \c PyCPPWrapperException exception.
/// This behavior is to convert failures in the python C API (such as
/// PyObject_New() returning NULL) to C++ exception so that we can unify
/// error handling in the style of C++ exceptions.
///
/// Examples 1: To create a tuple of two python objects, do this:
///
/// \code
/// try {
/// PyObjectContainer container0(Py_BuildValue("I", 0));
/// PyObjectContainer container1(Py_BuildValue("s", cppobj.toText().c_str()));
/// return (Py_BuildValue("OO", container0.get(), container1.get()));
/// } catch { ... set python exception, etc ... } \endcode
///
/// Commonly deployed buggy implementation to achieve this would be like this:
/// \code
/// return (Py_BuildValue("OO", Py_BuildValue("I", 0),
/// Py_BuildValue("s", cppobj.toText().c_str())));
/// \endcode
/// One clear bug of this code is that references to the element objects of
/// the tuple will leak.
/// (Assuming \c cppobj.toText() can throw) this code is also not exception
/// safe; if \c cppobj.toText() throws the reference to the first object
/// will leak, even if the code tried to do the necessary cleanup in the
/// successful case.
/// Further, this code naively passes the result of the first two calls to
/// \c Py_BuildValue() to the third one even if they can be NULL.
/// In this specific case, it happens to be okay because \c Py_BuildValue()
/// accepts NULL and treats it as an indication of error. But not all
/// python C library works that way (remember, the API is so inconsistent)
/// and we need to refer to the API manual every time we have to worry about
/// passing a NULL object to a library function. We'd certainly like to
/// avoid such development overhead. The code using \c PyObjectContainer
/// addresses all these problems.
///
/// Examples 2: Install a (constant) variable to a class.
///
/// \code
/// try {
/// // installClassVariable is a wrapper of
/// // PyObjectContainer::installAsClassVariable. See below.
/// installClassVariable(myclass_type, "SOME_CONSTANT",
/// Py_BuildValue("I", 0));
/// } catch { ... }
/// \endcode
///
/// Examples 3: Install a custom exception to a module.
///
/// \code
/// PyObject* new_exception; // publicly visible
/// ...
/// try {
/// new_exception = PyErr_NewException("mymodule.NewException",
/// NULL, NULL);
/// PyObjectContainer(new_exception).installToModule(mymodule,
/// "NewException");
/// } catch { ... }
/// \endcode
///
/// Note that \c installToModule() keeps the reference to \c new_exception
/// by default. This is a common practice when we introduce a custom
/// exception in a python biding written in C/C++. See the code comment
/// of the method for more details.
struct PyObjectContainer {
PyObjectContainer(PyObject* obj) : obj_(obj) {
if (obj_ == NULL) {
isc_throw(PyCPPWrapperException, "Unexpected NULL PyObject, "
"probably due to short memory");
}
}
virtual ~PyObjectContainer() {
if (obj_ != NULL) {
Py_DECREF(obj_);
}
}
PyObject* get() {
return (obj_);
}
PyObject* release() {
PyObject* ret = obj_;
obj_ = NULL;
return (ret);
}
// Install the enclosed PyObject to the specified python class 'pyclass'
// as a variable named 'name'.
void installAsClassVariable(PyTypeObject& pyclass, const char* name) {
if (PyDict_SetItemString(pyclass.tp_dict, name, obj_) < 0) {
isc_throw(PyCPPWrapperException, "Failed to set a class variable, "
"probably due to short memory");
}
// Ownership successfully transferred to the class object. We'll let
// it be released in the destructor.
}
// Install the enclosed PyObject to the specified module 'mod' as an
// object named 'name'.
// By default, this method explicitly keeps the reference to the object
// even after the module "steals" it. To cancel this behavior and give
// the reference to the module completely, the third parameter 'keep_ref'
// should be set to false.
void installToModule(PyObject* mod, const char* name,
bool keep_ref = true)
{
if (PyModule_AddObject(mod, name, obj_) < 0) {
isc_throw(PyCPPWrapperException, "Failed to add an object to "
"module, probably due to short memory");
}
// PyModule_AddObject has "stolen" the reference, so unless we
// have to retain it ourselves we don't (shouldn't) decrease it.
// However, we actually often need to keep our own reference because
// objects added to a module are often referenced via non local
// C/C++ variables in various places of the C/C++ code. In order
// for the code to run safely even if some buggy/evil python program
// performs 'del mod.obj', we need the extra reference. See, e.g.:
// http://docs.python.org/py3k/c-api/init.html#Py_Initialize
// http://mail.python.org/pipermail/python-dev/2005-June/054238.html
if (keep_ref) {
Py_INCREF(obj_);
}
obj_ = NULL;
}
protected:
PyObject* obj_;
};
/// This templated class is a derived class of \c PyObjectContainer and
/// manages C++-class based python objects.
///
/// The template parameter \c PYSTRUCT must be a derived class (structure) of
/// \c PyObject that has a member variable named \c cppobj, which must be a
/// a pointer to \c CPPCLASS (the second template parameter).
///
/// For example, to define a custom python class based on a C++ class, MyClass,
/// we'd define a class (struct) named \c s_MyClass like this:
/// \code
/// class s_MyClass : public PyObject {
/// public:
/// s_MyClass() : cppobj(NULL) {}
/// MyClass* cppobj;
/// };
/// \endcode
///
/// And, to build and return a python version of MyClass object, write the
/// following C++ code:
/// \code
/// typedef CPPPyObjectContainer<s_MyClass, MyClass> MyContainer;
/// try {
/// // below, myclass_type is of \c PyTypeObject that defines
/// // a python class (type) for MyClass
/// MyContainer container(PyObject_New(s_MyClass, myclass_type));
/// container.set(new MyClass());
/// return (container.release()); // give the reference to the caller
/// } catch { ... }
/// \endcode
///
/// This code prevents bugs like NULL pointer dereference when \c PyObject_New
/// fails or resource leaks when new'ing \c MyClass results in an exception.
/// Note that we use \c release() (derived from the base class) instead of
/// \c get(); in this case we should simply pass the reference generated in
/// \c PyObject_New() to the caller.
template <typename PYSTRUCT, typename CPPCLASS>
struct CPPPyObjectContainer : public PyObjectContainer {
CPPPyObjectContainer(PYSTRUCT* obj) : PyObjectContainer(obj) {}
// This method associates a C++ object with the corresponding python
// object enclosed in this class.
void set(CPPCLASS* value) {
if (value == NULL) {
isc_throw(PyCPPWrapperException, "Unexpected NULL C++ object, "
"probably due to short memory");
}
static_cast<PYSTRUCT*>(obj_)->cppobj = value;
}
// This is a convenience short cut to associate a C++ object with the
// python object and install it to the specified python class \c pyclass
// as a variable named \c name.
void installAsClassVariable(PyTypeObject& pyclass, const char* name,
CPPCLASS* value)
{
set(value);
PyObjectContainer::installAsClassVariable(pyclass, name);
}
};
/// A shortcut function to install a python class variable.
///
/// It installs a python object \c obj to a specified class \c pyclass
/// as a variable named \c name.
inline void
installClassVariable(PyTypeObject& pyclass, const char* name, PyObject* obj) {
PyObjectContainer(obj).installAsClassVariable(pyclass, name);
}
} // namespace python
} // namespace util
} // namespace isc
#endif // __PYCPPWRAPPER_UTIL_H
// Local Variables:
// mode: c++
// End:
// Copyright (C) @YEAR@ Internet Systems Consortium, Inc. ("ISC")
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
#include <string>
#include <stdexcept>
#include <util/python/pycppwrapper_util.h>
#include "@cppclass@_python.h"
using namespace std;
using namespace isc::util::python;
using namespace isc::@MODULE@;
using namespace isc::@MODULE@::python;
//
// Definition of the classes
//
// For each class, we need a struct, a helper functions (init, destroy,
// and static wrappers around the methods we export), a list of methods,
// and a type description
//
// @CPPCLASS@
//
// Trivial constructor.
s_@CPPCLASS@::s_@CPPCLASS@() : cppobj(NULL) {
}
namespace {
// Shortcut type which would be convenient for adding class variables safely.
typedef CPPPyObjectContainer<s_@CPPCLASS@, @CPPCLASS@> @CPPCLASS@Container;
//
// We declare the functions here, the definitions are below
// the type definition of the object, since both can use the other
//
// General creation and destruction
int @CPPCLASS@_init(s_@CPPCLASS@* self, PyObject* args);
void @CPPCLASS@_destroy(s_@CPPCLASS@* self);
// These are the functions we export
// ADD/REMOVE/MODIFY THE FOLLOWING AS APPROPRIATE FOR THE ACTUAL CLASS.
//
PyObject* @CPPCLASS@_toText(const s_@CPPCLASS@* const self);
PyObject* @CPPCLASS@_str(PyObject* self);
PyObject* @CPPCLASS@_richcmp(const s_@CPPCLASS@* const self,
const s_@CPPCLASS@* const other, int op);
// This is quite specific pydnspp. For other wrappers this should probably
// be removed.
PyObject* @CPPCLASS@_toWire(const s_@CPPCLASS@* self, PyObject* args);
// These are the functions we export
// For a minimal support, we don't need them.
// This list contains the actual set of functions we have in
// python. Each entry has
// 1. Python method name
// 2. Our static function here
// 3. Argument type
// 4. Documentation
PyMethodDef @CPPCLASS@_methods[] = {
{ "to_text", reinterpret_cast<PyCFunction>(@CPPCLASS@_toText), METH_NOARGS,
"Returns the text representation" },
// This is quite specific pydnspp. For other wrappers this should probably
// be removed:
{ "to_wire", reinterpret_cast<PyCFunction>(@CPPCLASS@_toWire), METH_VARARGS,
"Converts the @CPPCLASS@ object to wire format.\n"
"The argument can be either a MessageRenderer or an object that "
"implements the sequence interface. If the object is mutable "
"(for instance a bytearray()), the wire data is added in-place.\n"
"If it is not (for instance a bytes() object), a new object is "
"returned" },
{ NULL, NULL, 0, NULL }
};
// This is a template of typical code logic of python class initialization
// with C++ backend. You'll need to adjust it according to details of the
// actual C++ class.
int
@CPPCLASS@_init(s_@CPPCLASS@* self, PyObject* args) {
try {
if (PyArg_ParseTuple(args, "REPLACE ME")) {
// YOU'LL NEED SOME VALIDATION, PREPARATION, ETC, HERE.
self->cppobj = new @CPPCLASS@(/*NECESSARY PARAMS*/);
return (0);
}
} catch (const exception& ex) {
const string ex_what = "Failed to construct @CPPCLASS@ object: " +
string(ex.what());
PyErr_SetString(po_IscException, ex_what.c_str());
} catch (...) {
PyErr_SetString(po_IscException,
"Unexpected exception in constructing @CPPCLASS@");
return (-1);
}
PyErr_SetString(PyExc_TypeError,
"Invalid arguments to @CPPCLASS@ constructor");
return (-1);
}
// This is a template of typical code logic of python object destructor.
// In many cases you can use it without modification, but check that carefully.
void
@CPPCLASS@_destroy(s_@CPPCLASS@* const self) {
delete self->cppobj;
self->cppobj = NULL;
Py_TYPE(self)->tp_free(self);
}
// This should be able to be used without modification as long as the
// underlying C++ class has toText().
PyObject*
@CPPCLASS@_toText(const s_@CPPCLASS@* const self) {
try {
// toText() could throw, so we need to catch any exceptions below.
return (Py_BuildValue("s", self->cppobj->toText().c_str()));
} catch (const exception& ex) {
const string ex_what =
"Failed to convert @CPPCLASS@ object to text: " +
string(ex.what());
PyErr_SetString(po_IscException, ex_what.c_str());
} catch (...) {
PyErr_SetString(PyExc_SystemError, "Unexpected failure in "
"converting @CPPCLASS@ object to text");
}
return (NULL);
}
PyObject*
@CPPCLASS@_str(PyObject* self) {
// Simply call the to_text method we already defined
return (PyObject_CallMethod(self, const_cast<char*>("to_text"),
const_cast<char*>("")));
}
PyObject*
@CPPCLASS@_richcmp(const s_@CPPCLASS@* const self,
const s_@CPPCLASS@* const other,
const int op)
{
bool c = false;
// Check for null and if the types match. If different type,
// simply return False
if (other == NULL || (self->ob_type != other->ob_type)) {
Py_RETURN_FALSE;
}
// Only equals and not equals here, unorderable type
switch (op) {
case Py_LT:
PyErr_SetString(PyExc_TypeError, "Unorderable type; @CPPCLASS@");
return (NULL);
case Py_LE:
PyErr_SetString(PyExc_TypeError, "Unorderable type; @CPPCLASS@");
return (NULL);
case Py_EQ:
c = (*self->cppobj == *other->cppobj);
break;
case Py_NE:
c = (*self->cppobj != *other->cppobj);
break;
case Py_GT:
PyErr_SetString(PyExc_TypeError, "Unorderable type; @CPPCLASS@");
return (NULL);
case Py_GE:
PyErr_SetString(PyExc_TypeError, "Unorderable type; @CPPCLASS@");
return (NULL);
}
if (c) {
Py_RETURN_TRUE;
} else {
Py_RETURN_FALSE;
}
}
} // end of unnamed namespace
namespace isc {
namespace @MODULE@ {
namespace python {
// This defines the complete type for reflection in python and
// parsing of PyObject* to s_@CPPCLASS@
// Most of the functions are not actually implemented and NULL here.
PyTypeObject @cppclass@_type = {
PyVarObject_HEAD_INIT(NULL, 0)
"libdns_python.@CPPCLASS@",
sizeof(s_@CPPCLASS@), // tp_basicsize
0, // tp_itemsize
reinterpret_cast<destructor>(@CPPCLASS@_destroy), // tp_dealloc
NULL, // tp_print
NULL, // tp_getattr
NULL, // tp_setattr
NULL, // tp_reserved
NULL, // tp_repr