Commit 8d04ad0a authored by Thomas Markwalder's avatar Thomas Markwalder

[#15,!11] Schema upgrades for Kea 1.5, now update subnet ids in existing host reservations

src/bin/admin/tests/pgsql_tests.sh.in
src/bin/admin/tests/cql_tests.sh.in
src/bin/admin/tests/mysql_tests.sh.in
    Added functions that verify subnet id values are updated

src/share/database/scripts/cql/upgrade_2.0_to_3.0.sh.in
src/share/database/scripts/mysql/upgrade_6.0_to_7.0.sh.in
src/share/database/scripts/pgsql/upgrade_4.0_to_5.0.sh.in
    Added logic to update subnet id values in existing reservations
    and options
parent 245df6e6
......@@ -22,12 +22,17 @@ db_name="keatest"
# Set location of the kea-admin.
keaadmin=@abs_top_builddir@/src/bin/admin/kea-admin
cql_wipe() {
# Wipe the database.
cql_execute_script $db_scripts_dir/cql/dhcpdb_drop.cql
assert_eq 0 $? "drop table query failed, exit code %d, expected %d"
}
cql_lease_init_test() {
test_start "cql.init"
# Wipe the database.
cql_execute_script $db_scripts_dir/cql/dhcpdb_drop.cql
assert_eq 0 $? "drop table query failed, exit code %d, expected %d"
cql_wipe
# Create the database
$keaadmin lease-init cql -u $db_user -p $db_password -n $db_name -d $db_scripts_dir
......@@ -63,8 +68,7 @@ cql_lease_init_test() {
assert_eq 2 $? "kea-admin failed to deny lease-init, expected exit code: %d, actual: %d"
# Wipe the database.
cql_execute_script $db_scripts_dir/cql/dhcpdb_drop.cql
assert_eq 0 $? "drop table query failed, exit code %d, expected %d"
cql_wipe
test_finish 0
}
......@@ -73,8 +77,7 @@ cql_lease_version_test() {
test_start "cql.version"
# Wipe the database.
cql_execute_script $db_scripts_dir/cql/dhcpdb_drop.cql
assert_eq 0 $? "drop table query failed, exit code %d, expected %d"
cql_wipe
# Create the database.
$keaadmin lease-init cql -u $db_user -p $db_password -n $db_name -d $db_scripts_dir
......@@ -85,8 +88,7 @@ cql_lease_version_test() {
assert_str_eq "3.0" $version "Expected kea-admin to return %s, returned value was %s"
# Wipe the database.
cql_execute_script $db_scripts_dir/cql/dhcpdb_drop.cql
assert_eq 0 $? "drop table query failed, exit code %d, expected %d"
cql_wipe
test_finish 0
}
......@@ -95,8 +97,7 @@ cql_upgrade_test() {
test_start "cql.upgrade"
# Wipe the database.
cql_execute_script $db_scripts_dir/cql/dhcpdb_drop.cql
assert_eq 0 $? "drop table query failed, exit code %d, expected %d"
cql_wipe
# Initialize database to schema 1.0.
cql_execute_script @abs_top_srcdir@/src/bin/admin/tests/dhcpdb_create_1.0.cql
......@@ -106,8 +107,7 @@ cql_upgrade_test() {
assert_eq 0 $? "lease-upgrade failed, expected exit code: %d, actual: %d"
# Wipe the database.
cql_execute_script $db_scripts_dir/cql/dhcpdb_drop.cql
assert_eq 0 $? "drop table query failed, exit code %d, expected %d"
cql_wipe
test_finish 0
}
......@@ -136,8 +136,7 @@ cql_lease4_dump_test() {
fi
# Wipe the database.
cql_execute_script $db_scripts_dir/cql/dhcpdb_drop.cql
assert_eq 0 $? "drop table query failed, exit code %d, expected %d"
cql_wipe
# Create the database
$keaadmin lease-init cql -u $db_user -p $db_password -n $db_name -d $db_scripts_dir
......@@ -178,8 +177,7 @@ INSERT INTO lease4(address, hwaddr, client_id, valid_lifetime, expire, subnet_id
rm $output_file
# Wipe the database.
cql_execute_script $db_scripts_dir/cql/dhcpdb_drop.cql
assert_eq 0 $? "drop table query failed, exit code %d, expected %d"
cql_wipe
test_finish 0
}
......@@ -208,8 +206,7 @@ cql_lease6_dump_test() {
fi
# Wipe the database.
cql_execute_script $db_scripts_dir/cql/dhcpdb_drop.cql
assert_eq 0 $? "drop table query failed, exit code %d, expected %d"
cql_wipe
# Create the database.
$keaadmin lease-init cql -u $db_user -p $db_password -n $db_name -d $db_scripts_dir
......@@ -251,15 +248,117 @@ INSERT INTO lease6(address, duid, valid_lifetime, expire, subnet_id,\
rm $output_file
# Wipe the database.
cql_execute_script $db_scripts_dir/cql/dhcpdb_drop.cql
assert_eq 0 $? "drop table query failed, exit code %d, expected %d"
cql_wipe
test_finish 0
}
# Upgrades an existing schema to a target newer version
# param target_version - desired schema version as "major.minor"
cql_upgrade_schema_to_version() {
target_version=$1
# Check if the scripts directory exists at all.
if [ ! -d ${db_scripts_dir}/cql ]; then
log_error "Invalid scripts directory: ${db_scripts_dir}/mysql"
exit 1
fi
# Check if there are any files in it
num_files=$(find ${db_scripts_dir}/cql/upgrade*.sh -type f | wc -l)
if [ $num_files -eq 0 ]; then
log_error "No scripts in ${db_scripts_dir}/cql?"
exit 1
fi
version=`cql_version`
for script in ${db_scripts_dir}/cql/upgrade*.sh
do
if [ $version = "$target_version" ]
then
break;
fi
echo "Processing $script file..."
sh ${script} -u ${db_user} -p ${db_password} -k ${db_name}
version=`cql_version`
done
echo "Schema upgraded to $version"
}
# Verifies that you can upgrade from an earlier version and
# that unused subnet ID values in hosts and options tables are
# converted to -1
cql_unused_subnet_id_test() {
test_start "cql.unused_subnet_id_test"
# Let's wipe the whole database
cql_wipe
# We need to create an older database with lease data so we can
# verify the upgrade mechanisms which convert subnet id values
#
# Initialize database to scheme 1.0.
cql_execute_script @abs_top_srcdir@/src/bin/admin/tests/dhcpdb_create_1.0.cql
# Now upgrade to schema 2.0, the version just before global HRs
cql_upgrade_schema_to_version 2.0
# Now we need insert some hosts to "migrate" for both v4 and v6
qry=\
"insert into host_reservations (id, host_identifier_type, host_identifier, host_ipv4_subnet_id, host_ipv6_subnet_id, option_subnet_id)\
values (1, 0, textAsBlob('0123456'), 0, 0, 0);\
insert into host_reservations (id, host_identifier_type, host_identifier, host_ipv4_subnet_id, host_ipv6_subnet_id, option_subnet_id)\
values (2, 0, textAsBlob('1123456'), 4, 0, 4);\
insert into host_reservations (id, host_identifier_type, host_identifier, host_ipv4_subnet_id, host_ipv6_subnet_id, option_subnet_id)\
values (3, 0, textAsBlob('2123456'), 0, 6, 6);\
insert into host_reservations (id, host_identifier_type, host_identifier, host_ipv4_subnet_id, host_ipv6_subnet_id, option_subnet_id)\
values (4, 0, textAsBlob('3123456'), 4, 6, 0);\
insert into host_reservations (id, host_identifier_type, host_identifier, host_ipv4_subnet_id, host_ipv6_subnet_id, option_subnet_id)\
values (5, 0, textAsBlob('3123456'), -1, 6, 6);"
cql_execute "$qry"
assert_eq 0 $? "insert hosts failed, expected exit code: %d, actual: %d"
# Ok, we have a 2.0 database with hosts and options. Let's upgrade it.
${keaadmin} lease-upgrade cql -u $db_user -p $db_password -n $db_name -d $db_scripts_dir
ERRCODE=$?
# Upgrade should succeed
assert_eq 0 $ERRCODE "upgrade failed"
test_dir="@abs_top_srcdir@/src/bin/admin/tests"
ref_file="$test_dir/data/cql.subnet_id_test.reference.csv"
output_dir="@abs_top_builddir@/src/bin/admin/tests"
export_file="$output_dir/data/cql.subnet_id_test.csv"
# Fetch host_reservation data for comparison
echo "Exporting host_reservation data to $export_file ..."
qry=\
"select id, host_ipv4_subnet_id, host_ipv6_subnet_id, option_subnet_id from\
host_reservations where id in(1,2,3,4,5);"
cql_execute "$qry" >$export_file
assert_eq 0 $? "insert hosts failed, expected exit code: %d, actual: %d"
# Compare the dump output to reference file, they should be identical.
cmp -s $export_file $ref_file
assert_eq 0 $? "export file does not match reference file, expected exit code %d, actual %d"
# remove the output file.
rm $export_file
# Wipe the database.
cql_wipe
}
# Run tests.
cql_lease_init_test
cql_lease_version_test
cql_upgrade_test
cql_lease4_dump_test
cql_lease6_dump_test
cql_unused_subnet_id_test
......@@ -3,4 +3,5 @@ EXTRA_DIST = mysql.lease4_dump_test.reference.csv \
pgsql.lease4_dump_test.reference.csv \
pgsql.lease6_dump_test.reference.csv \
cql.lease4_dump_test.reference.csv \
cql.lease6_dump_test.reference.csv
cql.lease6_dump_test.reference.csv \
cql.subnet_id_test.reference.csv
id | host_ipv4_subnet_id | host_ipv6_subnet_id | option_subnet_id
----+---------------------+---------------------+------------------
1 | -1 | -1 | -1
2 | 4 | -1 | 4
3 | -1 | 6 | 6
4 | 4 | 6 | -1
5 | -1 | 6 | 6
(5 rows)
......@@ -869,6 +869,89 @@ mysql_lease_stat_upgrade_test() {
mysql_wipe
}
# Verifies that you can upgrade from an earlier version and
# that unused subnet ID values in hosts and options tables are
# converted to NULL.
mysql_unused_subnet_id_test() {
test_start "mysql.unused_subnet_id_test"
# Let's wipe the whole database
mysql_wipe
# We need to create an older database with lease data so we can
# verify the upgrade mechanisms which convert subnet id values
#
# Initialize database to scheme 1.0.
mysql -u$db_user -p$db_password $db_name < @abs_top_srcdir@/src/bin/admin/tests/dhcpdb_create_1.0.mysql
# Now upgrade to schema 6.0, this has lease_state in it
mysql_upgrade_schema_to_version 6.0
# Now we need insert some hosts to "migrate" for both v4 and v6
qry=\
"insert into hosts (dhcp_identifier_type, dhcp_identifier, dhcp4_subnet_id, dhcp6_subnet_id, hostname)\
values (0, '0123456', 0, 0, 'both'); \
insert into hosts (dhcp_identifier_type, dhcp_identifier, dhcp4_subnet_id, dhcp6_subnet_id, hostname)\
values (0, '1123456', 4, 0, 'v4only');
insert into hosts (dhcp_identifier_type, dhcp_identifier, dhcp4_subnet_id, dhcp6_subnet_id, hostname)\
values (0, '2123456', 0, 6, 'v6only');\
insert into hosts (dhcp_identifier_type, dhcp_identifier, dhcp4_subnet_id, dhcp6_subnet_id, hostname) \
values (0, '3123456', 4, 6, 'neither');"
run_statement "insert hosts" "$qry"
# Now we need insert some options to "migrate" for both v4 and v6
qry=\
"insert into dhcp4_options (code, dhcp4_subnet_id, scope_id) values (1, 4, 0);\
insert into dhcp4_options (code, dhcp4_subnet_id, scope_id) values (2, 0, 0);\
insert into dhcp6_options (code, dhcp6_subnet_id, scope_id) values (1, 6, 0);\
insert into dhcp6_options (code, dhcp6_subnet_id, scope_id) values (2, 0, 0);"
run_statement "insert options" "$qry"
# Ok, we have a 4.0 database with hosts and options. Let's upgrade it to 5.0
${keaadmin} lease-upgrade mysql -u $db_user -p $db_password -n $db_name -d $db_scripts_dir
ERRCODE=$?
# Upgrade should succeed
assert_eq 0 $ERRCODE "upgrade failed"
# Two hosts should have null v4 subnet ids
qry="select count(host_id) from hosts where dhcp4_subnet_id is null;"
run_statement "#hosts.1" "$qry" 2
# Two hosts should have v4 subnet ids = 4
qry="select count(host_id) from hosts where dhcp4_subnet_id = 4;"
run_statement "#hosts.2" "$qry" 2
# Two hosts should have null v6 subnet ids
qry="select count(host_id) from hosts where dhcp6_subnet_id is null;"
run_statement "#hosts.3" "$qry" 2
# Two hosts should should have v6 subnet ids = 6
qry="select count(host_id) from hosts where dhcp6_subnet_id = 6;"
run_statement "#hosts.4" "$qry" 2
# One option should have null v4 subnet id
qry="select count(option_id) from dhcp4_options where dhcp4_subnet_id is null;"
run_statement "#options.1" "$qry" 1
# One option should have v4 subnet id = 4
qry="select count(option_id) from dhcp4_options where dhcp4_subnet_id = 4;"
run_statement "#options.2" "$qry" 1
# One option should have null v6 subnet id
qry="select count(option_id) from dhcp6_options where dhcp6_subnet_id is null;"
run_statement "#options.3" "$qry" 1
# One option should have v4 subnet id = 6
qry="select count(option_id) from dhcp6_options where dhcp6_subnet_id = 6;"
run_statement "#options.4" "$qry" 1
# Let's wipe the whole database
mysql_wipe
}
mysql_lease_init_test
mysql_host_reservation_init_test
mysql_lease_version_test
......@@ -878,3 +961,4 @@ mysql_lease6_dump_test
mysql_lease4_stat_test
mysql_lease6_stat_test
mysql_lease_stat_upgrade_test
mysql_unused_subnet_id_test
......@@ -791,6 +791,92 @@ pgsql_lease_stat_upgrade_test() {
pgsql_wipe
}
# Verifies that you can upgrade from earlier version and
# that unused subnet ID values in hosts and options tables are
# converted to NULL.
pgsql_unused_subnet_id_test() {
test_start "pgsql.unused_subnet_id_test"
# Let's wipe the whole database
pgsql_wipe
# We need to create an older database with lease data so we can
# verify the upgrade mechanisms which prepopulate the lease stat
# tables.
#
# Initialize database to scheme 1.0.
pgsql_execute_script @abs_top_srcdir@/src/bin/admin/tests/dhcpdb_create_1.0.pgsql
assert_eq 0 $? "cannot initialize 1.0 database, expected exit code: %d, actual: %d"
# Now upgrade to schema 4.0
pgsql_upgrade_schema_to_version 4.0
# Now we need insert some hosts to "migrate" for both v4 and v6
qry=\
"insert into hosts (dhcp_identifier_type, dhcp_identifier, dhcp4_subnet_id, dhcp6_subnet_id, hostname)\
values (0, '0123456', 0, 0, 'both'); \
insert into hosts (dhcp_identifier_type, dhcp_identifier, dhcp4_subnet_id, dhcp6_subnet_id, hostname)\
values (0, '1123456', 4, 0, 'v4only');
insert into hosts (dhcp_identifier_type, dhcp_identifier, dhcp4_subnet_id, dhcp6_subnet_id, hostname)\
values (0, '2123456', 0, 6, 'v6only');\
insert into hosts (dhcp_identifier_type, dhcp_identifier, dhcp4_subnet_id, dhcp6_subnet_id, hostname) \
values (0, '3123456', 4, 6, 'neither');"
run_statement "insert hosts" "$qry"
# Now we need insert some options to "migrate" for both v4 and v6
qry=\
"insert into dhcp4_options (code, dhcp4_subnet_id, scope_id) values (1, 4, 0);\
insert into dhcp4_options (code, dhcp4_subnet_id, scope_id) values (2, 0, 0);\
insert into dhcp6_options (code, dhcp6_subnet_id, scope_id) values (1, 6, 0);\
insert into dhcp6_options (code, dhcp6_subnet_id, scope_id) values (2, 0, 0);"
run_statement "insert options" "$qry"
# Ok, we have a 4.0 database with hosts and options. Let's upgrade it to 5.0
${keaadmin} lease-upgrade pgsql -u $db_user -p $db_password -n $db_name -d $db_scripts_dir
ERRCODE=$?
# Upgrade should succeed
assert_eq 0 $ERRCODE "upgrade failed"
# Two hosts should have null v4 subnet ids
qry="select count(host_id) from hosts where dhcp4_subnet_id is null;"
run_statement "#hosts.1" "$qry" 2
# Two hosts should have v4 subnet ids = 4
qry="select count(host_id) from hosts where dhcp4_subnet_id = 4;"
run_statement "#hosts.2" "$qry" 2
# Two hosts should have null v6 subnet ids
qry="select count(host_id) from hosts where dhcp6_subnet_id is null;"
run_statement "#hosts.3" "$qry" 2
# Two hosts should should have v6 subnet ids = 6
qry="select count(host_id) from hosts where dhcp6_subnet_id = 6;"
run_statement "#hosts.4" "$qry" 2
# One option should have null v4 subnet id
qry="select count(option_id) from dhcp4_options where dhcp4_subnet_id is null;"
run_statement "#options.1" "$qry" 1
# One option should have v4 subnet id = 4
qry="select count(option_id) from dhcp4_options where dhcp4_subnet_id = 4;"
run_statement "#options.2" "$qry" 1
# One option should have null v6 subnet id
qry="select count(option_id) from dhcp6_options where dhcp6_subnet_id is null;"
run_statement "#options.3" "$qry" 1
# One option should have v4 subnet id = 6
qry="select count(option_id) from dhcp6_options where dhcp6_subnet_id = 6;"
run_statement "#options.4" "$qry" 1
# Let's wipe the whole database
pgsql_wipe
}
pgsql_lease_init_test
pgsql_lease_version_test
pgsql_upgrade_test
......@@ -799,3 +885,4 @@ pgsql_lease6_dump_test
pgsql_lease4_stat_test
pgsql_lease6_stat_test
pgsql_lease_stat_upgrade_test
pgsql_unused_subnet_id_test
......@@ -9,14 +9,28 @@ else
. @abs_top_builddir@/src/bin/admin/admin-utils.sh
fi
version=$(cql_version "$@")
if [ "${version}" != "2.0" ]; then
printf "This script upgrades 2.0 to 3.0. Reported version is %s. Skipping upgrade.\n" "${version}"
exit 0
# Need a path for temporary files created during upgrade data migration
# Use the state directory in the install path directory if it exists, otherwise
# use the build tree
if [ -e @localstatedir@/@PACKAGE_NAME@ ]; then
temp_file_dir="@localstatedir@/@PACKAGE_NAME@"
else
temp_file_dir="@abs_top_builddir@/src/share/database/scripts/cql"
fi
cqlsh "$@" <<EOF
cqlargs=$@
check_version() {
version=$(cql_version $cqlargs)
if [ "${version}" != "2.0" ]; then
printf "This script upgrades 2.0 to 3.0. Reported version is %s. Skipping upgrade.\n" "${version}"
exit 0
fi
}
update_schema() {
cqlsh $cqlargs <<EOF
-- This line starts database upgrade to version 3.0
-- Add a column holding leases for user context.
......@@ -46,4 +60,149 @@ INSERT INTO schema_version (version, minor) VALUES(3, 0);
-- This line concludes database upgrade to version 3.0
EOF
exit $?
if [ $? -ne 0 ]
then
echo Schema udpate FAILED!
exit -1
fi
}
clean_up() {
# clean up the files
if [ -e $export_file ]
then
rm $export_file
fi
if [ -e $update_file ]
then
rm $update_file
fi
}
exit_now() {
status=$1;shift
explanation=$1
clean_up
if [ $status -eq 0 ]
then
echo "Data Migration SUCCESS! $explanation"
else
echo "Data Migration FAILURE! $explanation"
fi
exit $status
}
check_column() {
local val="$1";shift
local col="$1"
local old_id="0"
local new_id="-1"
local comma=""
if [ "$val" = "$old_id" ]
then
if [ ! -z "$update_cols" ]
then
comma=","
fi
update_cols="$update_cols$comma $col = $new_id"
fi
}
migrate_host_data() {
export_file="$temp_file_dir/cql_export.csv"
update_file="$temp_file_dir/cql_update.cql"
clean_up
# Fetch host_reservation data so we have primary key, plus subnet id values
echo "Exporting host_reservation data to $export_file ..."
query="COPY host_reservations \
(id, host_ipv4_subnet_id, host_ipv6_subnet_id, option_subnet_id)\
TO '$export_file'"
cqlsh $cqlargs -e "$query"
if [ $? -ne 0 ]
then
exit_now -1 "Cassandra export failed! Could not migrate data!"
fi
# Iterate through the exported data, accumulating update statements,
# one for each reservation that needs updating. We should have one
# host per line.
line_cnt=0;
update_cnt=0;
while read line
do
let line_cnt++;
update_cols=""
xIFS="$IFS"
IFS=$'\r,'
i=1
# Parse the column values
for val in $line
do
case $i in
1)
host_id="$val"
;;
2)
check_column $val host_ipv4_subnet_id
;;
3)
check_column $val host_ipv6_subnet_id
;;
4)
check_column $val option_subnet_id
;;
*)
# We're going to assume that since any error is fatal
exit_now -1 "Line# $line_cnt, too many values, wrong or corrupt file"
;;
esac
let i++
done
if [ $i -ne 5 ]
then
# We're going to assume that since any error is fatal
exit_now -1 "Line# $line_cnt, too few values, wrong or corrupt file"
fi
# If any of the current host's columns need to be replace, append an update for it
if [ ! -z "$update_cols" ]
then
echo "update host_reservations set $update_cols where id = $host_id;" >> $update_file
let update_cnt++
fi
IFS="$xIFS"
done < $export_file
# If we didn't record any updates, then hey, we're good to go!
if [ $update_cnt == 0 ]
then
exit_now 0 "Completedly successfully: No updates were needed"
fi
# We have at least one update in the update file, so submit it # to cqlsh.
echo "$update_cnt update statements written to $update_file"
echo "Running the updates..."
cqlsh $cqlargs -f "$update_file"
if [ $? -ne 0 ]
then
exit_now -1 "Cassandra updates failed"
exit -1
fi
exit_now 0 "Updated $update_cnt of $line_cnt records"
}
check_version
update_schema
migrate_host_data
......@@ -106,6 +106,12 @@ CREATE INDEX timestamp_index ON logs (timestamp);
ALTER TABLE hosts
ADD COLUMN auth_key VARCHAR(16) NULL;
# Convert subnet-id values of 0 to NULL
UPDATE hosts SET dhcp4_subnet_id = NULL WHERE dhcp4_subnet_id = 0;
UPDATE dhcp4_options SET dhcp4_subnet_id = NULL WHERE dhcp4_subnet_id = 0;
UPDATE hosts SET dhcp6_subnet_id = NULL WHERE dhcp6_subnet_id = 0;
UPDATE dhcp6_options SET dhcp6_subnet_id = NULL WHERE dhcp6_subnet_id = 0;
# Update the schema version number
UPDATE schema_version
SET version = '7', minor = '0';
......
......@@ -136,6 +136,12 @@ CREATE INDEX address_id ON logs (address);
-- Create auth_key in hosts table for storing keys for DHCPv6 reconfigure.
ALTER TABLE hosts ADD COLUMN auth_key VARCHAR(16) DEFAULT NULL;
-- Convert subnet-id values of 0 to NULL
UPDATE hosts SET dhcp4_subnet_id = NULL WHERE dhcp4_subnet_id = 0;
UPDATE dhcp4_options SET dhcp4_subnet_id = NULL WHERE dhcp4_subnet_id = 0;
UPDATE hosts SET dhcp6_subnet_id = NULL WHERE dhcp6_subnet_id = 0;
UPDATE dhcp6_options SET dhcp6_subnet_id = NULL WHERE dhcp6_subnet_id = 0;
-- Set 5.0 schema version.
UPDATE schema_version
SET version = '5', minor = '0';
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment