upgrade_3.0_to_4.0.sh.in 10.7 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
#!/bin/sh

prefix=@prefix@
# Include utilities. Use installed version if available and
# use build version if it isn't.
if [ -e "@datarootdir@/@PACKAGE_NAME@/scripts/admin-utils.sh" ]; then
    . @datarootdir@/@PACKAGE_NAME@/scripts/admin-utils.sh
else
    . @abs_top_builddir@/src/bin/admin/admin-utils.sh
fi

# Need a path for temporary files created during upgrade data migration
13
# Use the state directory in the install path directory if it exists, otherwise
14
# use the build tree
15
16
if [ -e "@localstatedir@/lib/@PACKAGE_NAME@" ]; then
    temp_file_dir="@localstatedir@/lib/@PACKAGE_NAME@"
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
else
    temp_file_dir="@abs_top_builddir@/src/share/database/scripts/cql"
fi

cqlargs=$@

# Ensures the current schema version is 3.0. If not it exits.
check_version() {
    version=$(cql_version $cqlargs)

    if [ "${version}" != "3.0" ]; then
        printf "This script upgrades 3.0 to 4.0. Reported version is %s. Skipping upgrade.\n" "${version}"
        exit 0
    fi
}

# Peforms the schema changes from 3.0 to 4.0
update_schema() {
    cqlsh $cqlargs <<EOF
-- This line starts database upgrade to version 4.0

-- -----------------------------------------------------
-- Table \`hosts\`
-- -----------------------------------------------------
CREATE TABLE IF NOT EXISTS hosts (
    key BIGINT,
    id BIGINT,
    host_identifier BLOB,
    host_identifier_type INT,
    host_ipv4_subnet_id INT,
    host_ipv6_subnet_id INT,
    host_ipv4_address INT,
    host_ipv4_next_server INT,
    host_ipv4_server_hostname VARCHAR,
    host_ipv4_boot_file_name VARCHAR,
    hostname VARCHAR,
    auth_key VARCHAR,
    user_context VARCHAR,
    host_ipv4_client_classes VARCHAR,
    host_ipv6_client_classes VARCHAR,
    -- reservation
    reserved_ipv6_prefix_address VARCHAR,
    reserved_ipv6_prefix_length INT,
    reserved_ipv6_prefix_address_type INT,
    iaid INT,
    -- option
    option_universe INT,
    option_code INT,
    option_value BLOB,
    option_formatted_value VARCHAR,
    option_space VARCHAR,
    option_is_persistent BOOLEAN,
    option_client_class VARCHAR,
    option_subnet_id INT,
    option_user_context VARCHAR,
    option_scope_id INT,
    PRIMARY KEY ((key), id)
);

CREATE INDEX IF NOT EXISTS hostsindex1 ON hosts (host_identifier);
CREATE INDEX IF NOT EXISTS hostsindex2 ON hosts (host_identifier_type);
CREATE INDEX IF NOT EXISTS hostsindex3 ON hosts (host_ipv4_subnet_id);
CREATE INDEX IF NOT EXISTS hostsindex4 ON hosts (host_ipv6_subnet_id);
CREATE INDEX IF NOT EXISTS hostsindex5 ON hosts (host_ipv4_address);
CREATE INDEX IF NOT EXISTS hostsindex6 ON hosts (reserved_ipv6_prefix_address);
CREATE INDEX IF NOT EXISTS hostsindex7 ON hosts (reserved_ipv6_prefix_length);
EOF

    if [ "$?" -ne 0 ]
    then
        echo Schema udpate FAILED!
        exit 1
    fi
}

# Peforms the clean up schema changes from 3.0 to 4.0
clean_up_schema() {
    cqlsh $cqlargs <<EOF
DROP TABLE IF EXISTS host_reservations;

DROP INDEX IF EXISTS host_reservationsindex1;
DROP INDEX IF EXISTS host_reservationsindex2;
DROP INDEX IF EXISTS host_reservationsindex3;
DROP INDEX IF EXISTS host_reservationsindex4;
DROP INDEX IF EXISTS host_reservationsindex5;
DROP INDEX IF EXISTS host_reservationsindex6;
DROP INDEX IF EXISTS host_reservationsindex7;

-- Cql requires primary keys in the WHERE here.
DELETE FROM schema_version WHERE version=3;
INSERT INTO schema_version (version, minor) VALUES(4, 0);

-- This line concludes database upgrade to version 4.0
EOF

    if [ "$?" -ne 0 ]
    then
        echo Schema udpate FAILED!
        exit 1
    fi
}

# Function to delete temporary migration files
clean_up() {
    # clean up the files
    if [ -e "$export_file" ]
    then
        rm $export_file
    fi

    if [ -e "$update_file" ]
    then
        rm $update_file
    fi
}

# Function to clean up and exit the script gracefully
#
# Called by migrate_host_data()
#
# Parameters:
# status - integer value to pass to sh:exit
# explanation - "quoted" text message to emit to stdout
exit_now() {
    status=$1;shift
    explanation=$1

    clean_up
    if [ "$status" -eq 0 ]
    then
        clean_up_schema
        echo "Data Migration SUCCESS! $explanation"
    else
        echo "Data Migration FAILURE! $explanation"
    fi

    exit $status
}

fill() {
    string=$1;shift
    count=$1;shift
    fill_char=$1;shift
    length=`echo $string | wc -c`
Razvan Becheriu's avatar
Razvan Becheriu committed
161
    length=$((length - 1))
162
163
164
165
166
    if [ $length -gt $count ]; then
        value=`echo "$string" | cut -c 1-$count`
        return
    fi
    result=""
Razvan Becheriu's avatar
Razvan Becheriu committed
167
    count=$((count - length))
168
169
170
171
172
173
174
175
176
177
178
    i=1
    while [ $i -le $count ]; do
        result="$fill_char$result"
        i=$((i+1))
    done
    value="$result$string"
}

identifier_text() {
    string=$1;shift
    length=`echo $string | wc -c`
Razvan Becheriu's avatar
Razvan Becheriu committed
179
    length=$((length - 1))
180
181
182
183
    # skip 0x from 0xabcdef
    string=`echo "$string" | cut -c 3-$length`
    identifier=""
    # add starting 0: 0xabc->0x0abc
Razvan Becheriu's avatar
Razvan Becheriu committed
184
    mod=$((length % 2))
185
186
187
188
    if [ $mod -ne 0 ]; then
        string="0"$string
    fi
    length=`echo $string | wc -c`
Razvan Becheriu's avatar
Razvan Becheriu committed
189
    length=$((length - 1))
190
191
192
    i=1
    while [ $i -le $length ]; do
        char=`echo "$string" | cut -c $i-$i`
Razvan Becheriu's avatar
Razvan Becheriu committed
193
        mod=$((i % 2))
194
195
196
197
198
199
200
201
202
203
204
        if [ $mod -ne 0 -a $i -ne 1 ]; then
            char=":"$char
        fi
        identifier=$identifier$char
        i=$((i+1))
    done
}

key_hash() {
    string=$1;shift
    length=`echo $string | wc -c`
Razvan Becheriu's avatar
Razvan Becheriu committed
205
    length=$((length - 1))
206
207
208
209
210
211
212
    FNV_prime=1099511628211
    FNV_offset_basis=-3750763034362895579 # signed value for 14695981039346656037
    hash=$FNV_offset_basis
    i=1
    while [ $i -le $length ]; do
        char=`echo "$string" | cut -c $i-$i`
        data=`echo "$char" | tr -d "\n" | od -An -t uC | tr -d ' '`
Razvan Becheriu's avatar
Razvan Becheriu committed
213
214
        hash=$((hash ^ data))
        hash=$((hash * FNV_prime))
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
        i=$((i+1))
    done
}

generate_key() {
    host_id=$1;shift
    host_identifier=$1;shift
    host_identifier_type=$1;shift
    host_ipv4_subnet_id=$1;shift
    host_ipv6_subnet_id=$1;shift
    host_ipv4_address=$1;shift
    key=""
    identifier_text "$host_identifier"
    local_host_identifier=$identifier
    if [ ! -z $host_ipv4_address ] && [ $host_ipv4_address -eq 0 ]; then
        fill "$local_host_identifier" 383 "-"
        key="$key$value"
        fill "$host_identifier_type" 10 "-"
        key="$key$value"
    else
        fill "" 383 "-"
        key="$key$value"
        fill "" 10 "-"
        key="$key$value"
    fi
    fill "$host_ipv4_subnet_id" 10 "-"
    key="$key$value"
    fill "$host_ipv6_subnet_id" 10 "-"
    key="$key$value"
    fill "$host_ipv4_address" 15 "-"
    key="$key$value"
    key_hash "$key"
    key="$hash"
}

# This function adds host 'key' column which is the partition key
# of the 'hosts' table.
#
# After exhausting the export file, the update file is submitted to
# cqlsh for execution.
#
# No parameters.
migrate_host_data() {
    export_file="$temp_file_dir/cql_export.csv"
    update_file="$temp_file_dir/cql_update.cql"

    clean_up

    # Fetch host_reservation data so we have host_identifier,
    # host_identifier_type, host_ipv4_subnet_id, host_ipv6_subnet_id and
    # host_ipv4_address to generate host key
    echo "Exporting host_reservation data to $export_file ..."
    query="COPY host_reservations \
        (id, host_identifier, host_identifier_type, host_ipv4_subnet_id, \
         host_ipv6_subnet_id, host_ipv4_address, host_ipv4_next_server, \
         host_ipv4_server_hostname, host_ipv4_boot_file_name, hostname, \
         auth_key, user_context, host_ipv4_client_classes, \
         host_ipv6_client_classes, reserved_ipv6_prefix_address, \
         reserved_ipv6_prefix_length, reserved_ipv6_prefix_address_type, \
         iaid, option_universe, option_code, option_value, \
         option_formatted_value, option_space, option_is_persistent, \
         option_client_class, option_subnet_id, option_user_context, \
         option_scope_id) \
        TO '$export_file'"

    cqlsh $cqlargs -e "$query"
    if [ "$?" -ne 0 ]
    then
        exit_now 1 "Cassandra export failed! Could not migrate data!"
    fi

    # Strip the carriage returns that CQL insists on adding.
    if [ -e "$export_file" ]
    then
        cat $export_file | tr -d '\015' > $export_file.2
        mv $export_file.2 $export_file
    else
        # Shouldn't happen but then again we're talking about CQL here
        exit_now 1 "Cassandra export file $export_file is missing?"
    fi

    # Iterate through the exported data, accumulating update statements,
    # one for each reservation that needs updating.  We should have one
    # host per line.
    line_cnt=0;
    update_cnt=0;

    while read -r line
    do
        line_cnt=$((line_cnt + 1));
        xIFS="$IFS"
        IFS=','
        i=1
        # Parse the column values
        for val in $line
        do
            case $i in
            1)
                host_id=$val
                ;;
            2)
                host_identifier=$val
                ;;
            3)
                host_identifier_type=$val
                ;;
            4)
                host_ipv4_subnet_id=$val
                ;;
            5)
                host_ipv6_subnet_id=$val
                ;;
            6)
                host_ipv4_address=$val
                ;;
            *)
                ;;
            esac
            i=$((i + 1))
        done

        generate_key "$host_id" "$host_identifier" "$host_identifier_type" "$host_ipv4_subnet_id" "$host_ipv6_subnet_id" "$host_ipv4_address"
        key_data="$key"
        update_cnt=$((update_cnt + 1))

        IFS="$xIFS"
        echo $line | sed -e "s/$host_id/$host_id,$key_data/" >> $update_file
    done <  $export_file

    # If we didn't record any updates, then hey, we're good to go!
    if [ "$update_cnt" -eq 0 ]
    then
        exit_now 0 "Completed successfully: No updates were needed"
    fi

    # We have at least one update in the update file, so submit it # to cqlsh.
    echo "$update_cnt update statements written to $update_file"
    echo "Running the updates..."
    query="COPY hosts \
        (id, key, host_identifier, host_identifier_type, host_ipv4_subnet_id, \
         host_ipv6_subnet_id, host_ipv4_address, host_ipv4_next_server, \
         host_ipv4_server_hostname, host_ipv4_boot_file_name, hostname, \
         auth_key, user_context, host_ipv4_client_classes, \
         host_ipv6_client_classes, reserved_ipv6_prefix_address, \
         reserved_ipv6_prefix_length, reserved_ipv6_prefix_address_type, \
         iaid, option_universe, option_code, option_value, \
         option_formatted_value, option_space, option_is_persistent, \
         option_client_class, option_subnet_id, option_user_context, \
         option_scope_id) \
        FROM '$update_file'"

    cqlsh $cqlargs -e "$query"
    if [ "$?" -ne 0 ]
    then
        exit_now 1 "Cassandra updates failed"
    fi

    exit_now 0 "Updated $update_cnt of $line_cnt records"
}

check_version
update_schema
migrate_host_data