Commit 25b54fd9 authored by Thomas Markwalder's avatar Thomas Markwalder
Browse files

[master] Added reservation subnet-id conversion to Kea 1.5.0 schema upgrade

    Merge branch '15-global-host-reservations-task-5-data-migration-scripts-to-convert-existing-subnet-id-values-of-0'
parents a9e9e42d 39b3aa8b
......@@ -22,12 +22,17 @@ db_name="keatest"
# Set location of the kea-admin.
keaadmin=@abs_top_builddir@/src/bin/admin/kea-admin
cql_wipe() {
# Wipe the database.
cql_execute_script $db_scripts_dir/cql/dhcpdb_drop.cql
assert_eq 0 $? "drop table query failed, exit code %d, expected %d"
}
cql_lease_init_test() {
test_start "cql.init"
# Wipe the database.
cql_execute_script $db_scripts_dir/cql/dhcpdb_drop.cql
assert_eq 0 $? "drop table query failed, exit code %d, expected %d"
cql_wipe
# Create the database
$keaadmin lease-init cql -u $db_user -p $db_password -n $db_name -d $db_scripts_dir
......@@ -63,8 +68,7 @@ cql_lease_init_test() {
assert_eq 2 $? "kea-admin failed to deny lease-init, expected exit code: %d, actual: %d"
# Wipe the database.
cql_execute_script $db_scripts_dir/cql/dhcpdb_drop.cql
assert_eq 0 $? "drop table query failed, exit code %d, expected %d"
cql_wipe
test_finish 0
}
......@@ -73,8 +77,7 @@ cql_lease_version_test() {
test_start "cql.version"
# Wipe the database.
cql_execute_script $db_scripts_dir/cql/dhcpdb_drop.cql
assert_eq 0 $? "drop table query failed, exit code %d, expected %d"
cql_wipe
# Create the database.
$keaadmin lease-init cql -u $db_user -p $db_password -n $db_name -d $db_scripts_dir
......@@ -85,8 +88,7 @@ cql_lease_version_test() {
assert_str_eq "3.0" $version "Expected kea-admin to return %s, returned value was %s"
# Wipe the database.
cql_execute_script $db_scripts_dir/cql/dhcpdb_drop.cql
assert_eq 0 $? "drop table query failed, exit code %d, expected %d"
cql_wipe
test_finish 0
}
......@@ -95,8 +97,7 @@ cql_upgrade_test() {
test_start "cql.upgrade"
# Wipe the database.
cql_execute_script $db_scripts_dir/cql/dhcpdb_drop.cql
assert_eq 0 $? "drop table query failed, exit code %d, expected %d"
cql_wipe
# Initialize database to schema 1.0.
cql_execute_script @abs_top_srcdir@/src/bin/admin/tests/dhcpdb_create_1.0.cql
......@@ -106,8 +107,7 @@ cql_upgrade_test() {
assert_eq 0 $? "lease-upgrade failed, expected exit code: %d, actual: %d"
# Wipe the database.
cql_execute_script $db_scripts_dir/cql/dhcpdb_drop.cql
assert_eq 0 $? "drop table query failed, exit code %d, expected %d"
cql_wipe
test_finish 0
}
......@@ -136,8 +136,7 @@ cql_lease4_dump_test() {
fi
# Wipe the database.
cql_execute_script $db_scripts_dir/cql/dhcpdb_drop.cql
assert_eq 0 $? "drop table query failed, exit code %d, expected %d"
cql_wipe
# Create the database
$keaadmin lease-init cql -u $db_user -p $db_password -n $db_name -d $db_scripts_dir
......@@ -178,8 +177,7 @@ INSERT INTO lease4(address, hwaddr, client_id, valid_lifetime, expire, subnet_id
rm $output_file
# Wipe the database.
cql_execute_script $db_scripts_dir/cql/dhcpdb_drop.cql
assert_eq 0 $? "drop table query failed, exit code %d, expected %d"
cql_wipe
test_finish 0
}
......@@ -208,8 +206,7 @@ cql_lease6_dump_test() {
fi
# Wipe the database.
cql_execute_script $db_scripts_dir/cql/dhcpdb_drop.cql
assert_eq 0 $? "drop table query failed, exit code %d, expected %d"
cql_wipe
# Create the database.
$keaadmin lease-init cql -u $db_user -p $db_password -n $db_name -d $db_scripts_dir
......@@ -251,15 +248,117 @@ INSERT INTO lease6(address, duid, valid_lifetime, expire, subnet_id,\
rm $output_file
# Wipe the database.
cql_execute_script $db_scripts_dir/cql/dhcpdb_drop.cql
assert_eq 0 $? "drop table query failed, exit code %d, expected %d"
cql_wipe
test_finish 0
}
# Upgrades an existing schema to a target newer version
# param target_version - desired schema version as "major.minor"
cql_upgrade_schema_to_version() {
target_version=$1
# Check if the scripts directory exists at all.
if [ ! -d ${db_scripts_dir}/cql ]; then
log_error "Invalid scripts directory: ${db_scripts_dir}/mysql"
exit 1
fi
# Check if there are any files in it
num_files=$(find ${db_scripts_dir}/cql/upgrade*.sh -type f | wc -l)
if [ $num_files -eq 0 ]; then
log_error "No scripts in ${db_scripts_dir}/cql?"
exit 1
fi
version=`cql_version`
for script in ${db_scripts_dir}/cql/upgrade*.sh
do
if [ $version = "$target_version" ]
then
break;
fi
echo "Processing $script file..."
sh ${script} -u ${db_user} -p ${db_password} -k ${db_name}
version=`cql_version`
done
echo "Schema upgraded to $version"
}
# Verifies that you can upgrade from an earlier version and
# that unused subnet ID values in hosts and options tables are
# converted to -1
cql_unused_subnet_id_test() {
test_start "cql.unused_subnet_id_test"
# Let's wipe the whole database
cql_wipe
# We need to create an older database with lease data so we can
# verify the upgrade mechanisms which convert subnet id values
#
# Initialize database to schema 1.0.
cql_execute_script @abs_top_srcdir@/src/bin/admin/tests/dhcpdb_create_1.0.cql
# Now upgrade to schema 2.0, the version just before global HRs
cql_upgrade_schema_to_version 2.0
# Now we need insert some hosts to "migrate" for both v4 and v6
qry=\
"insert into host_reservations (id, host_identifier_type, host_identifier, host_ipv4_subnet_id, host_ipv6_subnet_id, option_subnet_id)\
values (1, 0, textAsBlob('0123456'), 0, 0, 0);\
insert into host_reservations (id, host_identifier_type, host_identifier, host_ipv4_subnet_id, host_ipv6_subnet_id, option_subnet_id)\
values (2, 0, textAsBlob('1123456'), 4, 0, 4);\
insert into host_reservations (id, host_identifier_type, host_identifier, host_ipv4_subnet_id, host_ipv6_subnet_id, option_subnet_id)\
values (3, 0, textAsBlob('2123456'), 0, 6, 6);\
insert into host_reservations (id, host_identifier_type, host_identifier, host_ipv4_subnet_id, host_ipv6_subnet_id, option_subnet_id)\
values (4, 0, textAsBlob('3123456'), 4, 6, 0);\
insert into host_reservations (id, host_identifier_type, host_identifier, host_ipv4_subnet_id, host_ipv6_subnet_id, option_subnet_id)\
values (5, 0, textAsBlob('3123456'), -1, 6, 6);"
cql_execute "$qry"
assert_eq 0 $? "insert hosts failed, expected exit code: %d, actual: %d"
# Ok, we have a 2.0 database with hosts and options. Let's upgrade it.
${keaadmin} lease-upgrade cql -u $db_user -p $db_password -n $db_name -d $db_scripts_dir
ERRCODE=$?
# Upgrade should succeed
assert_eq 0 $ERRCODE "upgrade failed"
test_dir="@abs_top_srcdir@/src/bin/admin/tests"
ref_file="$test_dir/data/cql.subnet_id_test.reference.csv"
output_dir="@abs_top_builddir@/src/bin/admin/tests"
export_file="$output_dir/data/cql.subnet_id_test.csv"
# Fetch host_reservation data for comparison
echo "Exporting host_reservation data to $export_file ..."
qry=\
"select id, host_ipv4_subnet_id, host_ipv6_subnet_id, option_subnet_id from\
host_reservations where id in(1,2,3,4,5);"
cql_execute "$qry" >$export_file
assert_eq 0 $? "insert hosts failed, expected exit code: %d, actual: %d"
# Compare the dump output to reference file, they should be identical.
cmp -s $export_file $ref_file
assert_eq 0 $? "export file does not match reference file, expected exit code %d, actual %d"
# remove the output file.
rm $export_file
# Wipe the database.
cql_wipe
}
# Run tests.
cql_lease_init_test
cql_lease_version_test
cql_upgrade_test
cql_lease4_dump_test
cql_lease6_dump_test
cql_unused_subnet_id_test
......@@ -3,4 +3,5 @@ EXTRA_DIST = mysql.lease4_dump_test.reference.csv \
pgsql.lease4_dump_test.reference.csv \
pgsql.lease6_dump_test.reference.csv \
cql.lease4_dump_test.reference.csv \
cql.lease6_dump_test.reference.csv
cql.lease6_dump_test.reference.csv \
cql.subnet_id_test.reference.csv
id | host_ipv4_subnet_id | host_ipv6_subnet_id | option_subnet_id
----+---------------------+---------------------+------------------
1 | -1 | -1 | -1
2 | 4 | -1 | 4
3 | -1 | 6 | 6
4 | 4 | 6 | -1
5 | -1 | 6 | 6
(5 rows)
......@@ -869,6 +869,89 @@ mysql_lease_stat_upgrade_test() {
mysql_wipe
}
# Verifies that you can upgrade from an earlier version and
# that unused subnet ID values in hosts and options tables are
# converted to NULL.
mysql_unused_subnet_id_test() {
test_start "mysql.unused_subnet_id_test"
# Let's wipe the whole database
mysql_wipe
# We need to create an older database with lease data so we can
# verify the upgrade mechanisms which convert subnet id values
#
# Initialize database to scheme 1.0.
mysql -u$db_user -p$db_password $db_name < @abs_top_srcdir@/src/bin/admin/tests/dhcpdb_create_1.0.mysql
# Now upgrade to schema 6.0, this has lease_state in it
mysql_upgrade_schema_to_version 6.0
# Now we need insert some hosts to "migrate" for both v4 and v6
qry=\
"insert into hosts (dhcp_identifier_type, dhcp_identifier, dhcp4_subnet_id, dhcp6_subnet_id, hostname)\
values (0, '0123456', 0, 0, 'both'); \
insert into hosts (dhcp_identifier_type, dhcp_identifier, dhcp4_subnet_id, dhcp6_subnet_id, hostname)\
values (0, '1123456', 4, 0, 'v4only');
insert into hosts (dhcp_identifier_type, dhcp_identifier, dhcp4_subnet_id, dhcp6_subnet_id, hostname)\
values (0, '2123456', 0, 6, 'v6only');\
insert into hosts (dhcp_identifier_type, dhcp_identifier, dhcp4_subnet_id, dhcp6_subnet_id, hostname) \
values (0, '3123456', 4, 6, 'neither');"
run_statement "insert hosts" "$qry"
# Now we need insert some options to "migrate" for both v4 and v6
qry=\
"insert into dhcp4_options (code, dhcp4_subnet_id, scope_id) values (1, 4, 0);\
insert into dhcp4_options (code, dhcp4_subnet_id, scope_id) values (2, 0, 0);\
insert into dhcp6_options (code, dhcp6_subnet_id, scope_id) values (1, 6, 0);\
insert into dhcp6_options (code, dhcp6_subnet_id, scope_id) values (2, 0, 0);"
run_statement "insert options" "$qry"
# Ok, we have a 4.0 database with hosts and options. Let's upgrade it to 5.0
${keaadmin} lease-upgrade mysql -u $db_user -p $db_password -n $db_name -d $db_scripts_dir
ERRCODE=$?
# Upgrade should succeed
assert_eq 0 $ERRCODE "upgrade failed"
# Two hosts should have null v4 subnet ids
qry="select count(host_id) from hosts where dhcp4_subnet_id is null;"
run_statement "#hosts.1" "$qry" 2
# Two hosts should have v4 subnet ids = 4
qry="select count(host_id) from hosts where dhcp4_subnet_id = 4;"
run_statement "#hosts.2" "$qry" 2
# Two hosts should have null v6 subnet ids
qry="select count(host_id) from hosts where dhcp6_subnet_id is null;"
run_statement "#hosts.3" "$qry" 2
# Two hosts should should have v6 subnet ids = 6
qry="select count(host_id) from hosts where dhcp6_subnet_id = 6;"
run_statement "#hosts.4" "$qry" 2
# One option should have null v4 subnet id
qry="select count(option_id) from dhcp4_options where dhcp4_subnet_id is null;"
run_statement "#options.1" "$qry" 1
# One option should have v4 subnet id = 4
qry="select count(option_id) from dhcp4_options where dhcp4_subnet_id = 4;"
run_statement "#options.2" "$qry" 1
# One option should have null v6 subnet id
qry="select count(option_id) from dhcp6_options where dhcp6_subnet_id is null;"
run_statement "#options.3" "$qry" 1
# One option should have v4 subnet id = 6
qry="select count(option_id) from dhcp6_options where dhcp6_subnet_id = 6;"
run_statement "#options.4" "$qry" 1
# Let's wipe the whole database
mysql_wipe
}
mysql_lease_init_test
mysql_host_reservation_init_test
mysql_lease_version_test
......@@ -878,3 +961,4 @@ mysql_lease6_dump_test
mysql_lease4_stat_test
mysql_lease6_stat_test
mysql_lease_stat_upgrade_test
mysql_unused_subnet_id_test
......@@ -791,6 +791,92 @@ pgsql_lease_stat_upgrade_test() {
pgsql_wipe
}
# Verifies that you can upgrade from earlier version and
# that unused subnet ID values in hosts and options tables are
# converted to NULL.
pgsql_unused_subnet_id_test() {
test_start "pgsql.unused_subnet_id_test"
# Let's wipe the whole database
pgsql_wipe
# We need to create an older database with lease data so we can
# verify the upgrade mechanisms which prepopulate the lease stat
# tables.
#
# Initialize database to scheme 1.0.
pgsql_execute_script @abs_top_srcdir@/src/bin/admin/tests/dhcpdb_create_1.0.pgsql
assert_eq 0 $? "cannot initialize 1.0 database, expected exit code: %d, actual: %d"
# Now upgrade to schema 4.0
pgsql_upgrade_schema_to_version 4.0
# Now we need insert some hosts to "migrate" for both v4 and v6
qry=\
"insert into hosts (dhcp_identifier_type, dhcp_identifier, dhcp4_subnet_id, dhcp6_subnet_id, hostname)\
values (0, '0123456', 0, 0, 'both'); \
insert into hosts (dhcp_identifier_type, dhcp_identifier, dhcp4_subnet_id, dhcp6_subnet_id, hostname)\
values (0, '1123456', 4, 0, 'v4only');
insert into hosts (dhcp_identifier_type, dhcp_identifier, dhcp4_subnet_id, dhcp6_subnet_id, hostname)\
values (0, '2123456', 0, 6, 'v6only');\
insert into hosts (dhcp_identifier_type, dhcp_identifier, dhcp4_subnet_id, dhcp6_subnet_id, hostname) \
values (0, '3123456', 4, 6, 'neither');"
run_statement "insert hosts" "$qry"
# Now we need insert some options to "migrate" for both v4 and v6
qry=\
"insert into dhcp4_options (code, dhcp4_subnet_id, scope_id) values (1, 4, 0);\
insert into dhcp4_options (code, dhcp4_subnet_id, scope_id) values (2, 0, 0);\
insert into dhcp6_options (code, dhcp6_subnet_id, scope_id) values (1, 6, 0);\
insert into dhcp6_options (code, dhcp6_subnet_id, scope_id) values (2, 0, 0);"
run_statement "insert options" "$qry"
# Ok, we have a 4.0 database with hosts and options. Let's upgrade it to 5.0
${keaadmin} lease-upgrade pgsql -u $db_user -p $db_password -n $db_name -d $db_scripts_dir
ERRCODE=$?
# Upgrade should succeed
assert_eq 0 $ERRCODE "upgrade failed"
# Two hosts should have null v4 subnet ids
qry="select count(host_id) from hosts where dhcp4_subnet_id is null;"
run_statement "#hosts.1" "$qry" 2
# Two hosts should have v4 subnet ids = 4
qry="select count(host_id) from hosts where dhcp4_subnet_id = 4;"
run_statement "#hosts.2" "$qry" 2
# Two hosts should have null v6 subnet ids
qry="select count(host_id) from hosts where dhcp6_subnet_id is null;"
run_statement "#hosts.3" "$qry" 2
# Two hosts should should have v6 subnet ids = 6
qry="select count(host_id) from hosts where dhcp6_subnet_id = 6;"
run_statement "#hosts.4" "$qry" 2
# One option should have null v4 subnet id
qry="select count(option_id) from dhcp4_options where dhcp4_subnet_id is null;"
run_statement "#options.1" "$qry" 1
# One option should have v4 subnet id = 4
qry="select count(option_id) from dhcp4_options where dhcp4_subnet_id = 4;"
run_statement "#options.2" "$qry" 1
# One option should have null v6 subnet id
qry="select count(option_id) from dhcp6_options where dhcp6_subnet_id is null;"
run_statement "#options.3" "$qry" 1
# One option should have v4 subnet id = 6
qry="select count(option_id) from dhcp6_options where dhcp6_subnet_id = 6;"
run_statement "#options.4" "$qry" 1
# Let's wipe the whole database
pgsql_wipe
}
pgsql_lease_init_test
pgsql_lease_version_test
pgsql_upgrade_test
......@@ -799,3 +885,4 @@ pgsql_lease6_dump_test
pgsql_lease4_stat_test
pgsql_lease6_stat_test
pgsql_lease_stat_upgrade_test
pgsql_unused_subnet_id_test
upgrade_1.0_to_2.0.sh
/upgrade_1.0_to_2.0.sh
/upgrade_2.0_to_3.0.sh
......@@ -9,14 +9,30 @@ else
. @abs_top_builddir@/src/bin/admin/admin-utils.sh
fi
version=$(cql_version "$@")
if [ "${version}" != "2.0" ]; then
printf "This script upgrades 2.0 to 3.0. Reported version is %s. Skipping upgrade.\n" "${version}"
exit 0
# Need a path for temporary files created during upgrade data migration
# Use the state directory in the install path directory if it exists, otherwise
# use the build tree
if [ -e @localstatedir@/@PACKAGE_NAME@ ]; then
temp_file_dir="@localstatedir@/@PACKAGE_NAME@"
else
temp_file_dir="@abs_top_builddir@/src/share/database/scripts/cql"
fi
cqlsh "$@" <<EOF
cqlargs=$@
# Ensures the current schema version is 2.0. If not it exits.
check_version() {
version=$(cql_version $cqlargs)
if [ "${version}" != "2.0" ]; then
printf "This script upgrades 2.0 to 3.0. Reported version is %s. Skipping upgrade.\n" "${version}"
exit 0
fi
}
# Peforms the schema changes from 2.0 to 3.0
update_schema() {
cqlsh $cqlargs <<EOF
-- This line starts database upgrade to version 3.0
-- Add a column holding leases for user context.
......@@ -46,4 +62,196 @@ INSERT INTO schema_version (version, minor) VALUES(3, 0);
-- This line concludes database upgrade to version 3.0
EOF
exit $?
if [ $? -ne 0 ]
then
echo Schema udpate FAILED!
exit -1
fi
}
# Function to delete temporary migration files
clean_up() {
# clean up the files
if [ -e $export_file ]
then
rm $export_file
fi
if [ -e $update_file ]
then
rm $update_file
fi
}
# Function to clean up and exit the script gracefully
#
# Called by migrate_host_data()
#
# Parameters:
# status - integer value to pass to sh:exit
# explanation - "quoted" text message to emit to stdout
exit_now() {
status=$1;shift
explanation=$1
clean_up
if [ $status -eq 0 ]
then
echo "Data Migration SUCCESS! $explanation"
else
echo "Data Migration FAILURE! $explanation"
fi
exit $status
}
# Function adds a column to the global, $update_cols if needed
#
# Called by migrate_host_data() to determine if the given column
# value needs to be updated, and if so appends CQL update text
# to a global string accumulator, $update_cols.
#
# The accumlator text is of the form:
#
# "<column_name>=<column value>{,<column_name>=<column_value>,..}"
#
# Parameters:
# val - current numerical value of the subnet ID column in question
# col - column name of the column in question
#
check_column() {
local val="$1";shift
local col="$1"
local old_id="0"
local new_id="-1"
local comma=""
# If the current value equals the value to be replaced
# add it to the accumulator
if [ "$val" = "$old_id" ]
then
# If the accumulator isn't empty, we need a comma
if [ ! -z "$update_cols" ]
then
comma=","
fi
update_cols="$update_cols$comma $col = $new_id"
fi
}
# This function converts subnet ID columns in of existing host_reservations
# from 0 to 0xFFFFFFFF (SUBNET_ID_UNUSED). To do this it first creates an
# excerpt from host_reservations containing one row for each reservation
# with the following columns:
#
# id (primary key)
# host_ipv4_subnet_id
# host_ipv6_subnet_id
# option_subnet_id
#
# The excerpt is captured in a temporary file, the "export" file.
#
# Next, it iterates over the export file emitting a CQL update statement
# for each row that needs at least one of the columns updated. In other
# words, if at least one of the subnet ID columns in a row is 0, an
# update statement for that row will be emitted. The update statements
# are captured in a second temprory file, the "update" file.
#
# After exhausting the export file, the update file is submitted to
# cqlsh for execution.
#
# No parameters.
migrate_host_data() {
export_file="$temp_file_dir/cql_export.csv"
update_file="$temp_file_dir/cql_update.cql"
clean_up
# Fetch host_reservation data so we have primary key, plus subnet id values
echo "Exporting host_reservation data to $export_file ..."
query="COPY host_reservations \
(id, host_ipv4_subnet_id, host_ipv6_subnet_id, option_subnet_id)\
TO '$export_file'"
cqlsh $cqlargs -e "$query"
if [ $? -ne 0 ]
then
exit_now -1 "Cassandra export failed! Could not migrate data!"
fi
# Iterate through the exported data, accumulating update statements,
# one for each reservation that needs updating. We should have one
# host per line.
line_cnt=0;
update_cnt=0;
while read line
do
let line_cnt++;
update_cols=""
xIFS="$IFS"
IFS=$'\r,'
i=1
# Parse the column values
for val in $line
do
case $i in
1)
host_id="$val"
;;
2)
check_column $val host_ipv4_subnet_id
;;
3)
check_column $val host_ipv6_subnet_id
;;
4)
check_column $val option_subnet_id
;;
*)