Commit 9b97fd86 authored by Michal 'vorner' Vaner's avatar Michal 'vorner' Vaner

Merge #2857: memory manager initial exchanges

This initializes the basic data structures of the memory manager and
starts loading data into the segments.
parents a0f6b0ff 5da41fa1
...@@ -29,7 +29,7 @@ from isc.log_messages.memmgr_messages import * ...@@ -29,7 +29,7 @@ from isc.log_messages.memmgr_messages import *
from isc.server_common.bind10_server import BIND10Server, BIND10ServerFatal from isc.server_common.bind10_server import BIND10Server, BIND10ServerFatal
from isc.server_common.datasrc_clients_mgr \ from isc.server_common.datasrc_clients_mgr \
import DataSrcClientsMgr, ConfigError import DataSrcClientsMgr, ConfigError
from isc.memmgr.datasrc_info import DataSrcInfo from isc.memmgr.datasrc_info import DataSrcInfo, SegmentInfo
from isc.memmgr.builder import MemorySegmentBuilder from isc.memmgr.builder import MemorySegmentBuilder
import isc.util.process import isc.util.process
...@@ -124,18 +124,51 @@ class Memmgr(BIND10Server): ...@@ -124,18 +124,51 @@ class Memmgr(BIND10Server):
# All copy, switch to the new configuration. # All copy, switch to the new configuration.
self._config_params = new_config_params self._config_params = new_config_params
def __notify_from_builder(self): def _cmd_to_builder(self, cmd):
# Nothing is implemented here for now. This method should have """
# code to handle responses from the builder in Send a command to the builder, with proper synchronization.
# self._builder_response_queue[]. Access must be synchronized """
# using self._builder_lock. assert isinstance(cmd, tuple)
pass with self._builder_cv:
def _notify_from_builder(self):
Read the notifications from the builder thread.
self._master_sock.recv(1) # Clear the wake-up data
notifications = None
with self._builder_lock:
# Copy the notifications out and clear them from the
# original list. We may not assign [] to
# self._builder_response_queue to clear it, because there's
# another reference to it from the other thread and it would
# not keep the original list.
notifications = self._builder_response_queue[:]
del self._builder_response_queue[:]
for notification in notifications:
notif_name = notification[0]
if notif_name == 'load-completed':
(_, dsrc_info, rrclass, dsrc_name) = notification
sgmt_info = dsrc_info.segment_info_map[(rrclass, dsrc_name)]
cmd = sgmt_info.complete_update()
# It may return another load command on the same data source.
# If it is so, we execute it too, before we start
# synchronizing with the readers.
if cmd is not None:
# TODO: Send to the readers, #2858
raise ValueError('Unknown notification name: ' + notif_name)
def __create_builder_thread(self): def __create_builder_thread(self):
# We get responses from the builder thread on this socket pair. # We get responses from the builder thread on this socket pair.
(self._master_sock, self._builder_sock) = \ (self._master_sock, self._builder_sock) = \
socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM) socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM)
self.watch_fileno(self._master_sock, rcallback=self.__notify_from_builder) self.watch_fileno(self._master_sock, rcallback=self._notify_from_builder)
# See the documentation for MemorySegmentBuilder on how the # See the documentation for MemorySegmentBuilder on how the
# following are used. # following are used.
...@@ -161,9 +194,7 @@ class Memmgr(BIND10Server): ...@@ -161,9 +194,7 @@ class Memmgr(BIND10Server):
# This makes the MemorySegmentBuilder exit its main loop. It # This makes the MemorySegmentBuilder exit its main loop. It
# should make the builder thread joinable. # should make the builder thread joinable.
with self._builder_cv: self._cmd_to_builder(('shutdown',))
self._builder_thread.join() self._builder_thread.join()
...@@ -202,13 +233,25 @@ class Memmgr(BIND10Server): ...@@ -202,13 +233,25 @@ class Memmgr(BIND10Server):
genid, clients_map = self._datasrc_clients_mgr.get_clients_map() genid, clients_map = self._datasrc_clients_mgr.get_clients_map()
datasrc_info = DataSrcInfo(genid, clients_map, self._config_params) datasrc_info = DataSrcInfo(genid, clients_map, self._config_params)
self._datasrc_info_list.append(datasrc_info) self._datasrc_info_list.append(datasrc_info)
# Full datasrc reconfig will be rare, so would be worth logging # Full datasrc reconfig will be rare, so would be worth logging
# at the info level. # at the info level., genid), genid)
except isc.server_common.datasrc_clients_mgr.ConfigError as ex: except isc.server_common.datasrc_clients_mgr.ConfigError as ex:
def _init_segments(self, datasrc_info):
for key, sgmt_info in datasrc_info.segment_info_map.items():
rrclass, dsrc_name = key
cmd = ('load', None, datasrc_info, rrclass, dsrc_name)
send_cmd = sgmt_info.start_update()
assert cmd == send_cmd and sgmt_info.get_state() == \
if '__main__' == __name__: if '__main__' == __name__:
mgr = Memmgr() mgr = Memmgr()
sys.exit( sys.exit(
...@@ -16,12 +16,14 @@ ...@@ -16,12 +16,14 @@
import unittest import unittest
import os import os
import re import re
import threading
import isc.log import isc.log
from isc.dns import RRClass from isc.dns import RRClass
import isc.config import isc.config
from isc.config import parse_answer from isc.config import parse_answer
import memmgr import memmgr
from isc.memmgr.datasrc_info import SegmentInfo
from isc.testutils.ccsession_mock import MockModuleCCSession from isc.testutils.ccsession_mock import MockModuleCCSession
class MyCCSession(MockModuleCCSession, isc.config.ConfigData): class MyCCSession(MockModuleCCSession, isc.config.ConfigData):
...@@ -190,9 +192,14 @@ class TestMemmgr(unittest.TestCase): ...@@ -190,9 +192,14 @@ class TestMemmgr(unittest.TestCase):
cfg_data = MockConfigData( cfg_data = MockConfigData(
{"classes": {"IN": [{"type": "MasterFiles", {"classes": {"IN": [{"type": "MasterFiles",
"cache-enable": True, "params": {}}]}}) "cache-enable": True, "params": {}}]}})
self.__init_called = None
def mock_init_segments(param):
self.__init_called = param
self.__mgr._init_segments = mock_init_segments
self.__mgr._datasrc_config_handler({}, cfg_data) self.__mgr._datasrc_config_handler({}, cfg_data)
self.assertEqual(1, len(self.__mgr._datasrc_info_list)) self.assertEqual(1, len(self.__mgr._datasrc_info_list))
self.assertEqual(1, self.__mgr._datasrc_info_list[0].gen_id) self.assertEqual(1, self.__mgr._datasrc_info_list[0].gen_id)
self.assertEqual(self.__init_called, self.__mgr._datasrc_info_list[0])
# Below we're using a mock DataSrcClientMgr for easier tests # Below we're using a mock DataSrcClientMgr for easier tests
class MockDataSrcClientMgr: class MockDataSrcClientMgr:
...@@ -220,6 +227,7 @@ class TestMemmgr(unittest.TestCase): ...@@ -220,6 +227,7 @@ class TestMemmgr(unittest.TestCase):
MockDataSrcClientMgr([('sqlite3', 'mapped', None)]) MockDataSrcClientMgr([('sqlite3', 'mapped', None)])
self.__mgr._datasrc_config_handler(None, None) # params don't matter self.__mgr._datasrc_config_handler(None, None) # params don't matter
self.assertEqual(2, len(self.__mgr._datasrc_info_list)) self.assertEqual(2, len(self.__mgr._datasrc_info_list))
self.assertEqual(self.__init_called, self.__mgr._datasrc_info_list[1])
self.assertIsNotNone( self.assertIsNotNone(
self.__mgr._datasrc_info_list[1].segment_info_map[ self.__mgr._datasrc_info_list[1].segment_info_map[
(RRClass.IN, 'sqlite3')]) (RRClass.IN, 'sqlite3')])
...@@ -230,6 +238,104 @@ class TestMemmgr(unittest.TestCase): ...@@ -230,6 +238,104 @@ class TestMemmgr(unittest.TestCase):
self.__mgr._datasrc_config_handler(None, None) self.__mgr._datasrc_config_handler(None, None)
self.assertEqual(2, len(self.__mgr._datasrc_info_list)) self.assertEqual(2, len(self.__mgr._datasrc_info_list))
def test_init_segments(self):
Test the initialization of segments ‒ just load everything found in there.
# Fake a lot of things. These are objects hard to set up, so this is
# easier.
class SgmtInfo:
def __init__(self): = []
self.__state = None
def add_event(self, cmd):
self.__state = SegmentInfo.UPDATING
def start_update(self):
def get_state(self):
return self.__state
sgmt_info = SgmtInfo()
class DataSrcInfo:
def __init__(self):
self.segment_info_map = \
{(isc.dns.RRClass.IN, "name"): sgmt_info}
dsrc_info = DataSrcInfo()
# Pretend to have the builder thread
self.__mgr._builder_cv = threading.Condition()
# Run the initialization
# The event was pushed into the segment info
command = ('load', None, dsrc_info, isc.dns.RRClass.IN, 'name')
self.assertEqual([command], self.__mgr._builder_command_queue)
del self.__mgr._builder_command_queue[:]
def test_notify_from_builder(self):
Check the notify from builder thing eats the notifications and
handles them.
# Some mocks
class SgmtInfo:
def complete_update():
return 'command'
sgmt_info = SgmtInfo
class DataSrcInfo:
def __init__(self):
self.segment_info_map = \
{(isc.dns.RRClass.IN, "name"): sgmt_info}
dsrc_info = DataSrcInfo()
class Sock:
def recv(self, size):
self.__mgr._master_sock = Sock()
commands = []
def mock_cmd_to_builder(cmd):
self.__mgr._cmd_to_builder = mock_cmd_to_builder
self.__mgr._builder_lock = threading.Lock()
# Extract the reference for the queue. We get a copy of the reference
# to check it is cleared, not a new empty one installed
notif_ref = self.__mgr._builder_response_queue
notif_ref.append(('load-completed', dsrc_info, isc.dns.RRClass.IN,
# Wake up the main thread and let it process the notifications
# All notifications are now eaten
self.assertEqual([], notif_ref)
self.assertEqual(['command'], commands)
del commands[:]
# The new command is sent
# Once again the same, but with the last command - nothing new pushed
sgmt_info.complete_update = lambda: None
notif_ref.append(('load-completed', dsrc_info, isc.dns.RRClass.IN,
self.assertEqual([], notif_ref)
self.assertEqual([], commands)
# This is invalid (unhandled) notification name
self.assertRaises(ValueError, self.__mgr._notify_from_builder)
self.assertEqual([], notif_ref)
def test_send_to_builder(self):
Send command to the builder test.
self.__mgr._builder_cv = threading.Condition()
self.assertEqual([('test',)], self.__mgr._builder_command_queue)
del self.__mgr._builder_command_queue[:]
if __name__== "__main__": if __name__== "__main__":
isc.log.resetUnitTestRootLogger() isc.log.resetUnitTestRootLogger()
unittest.main() unittest.main()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment