dhclient: concurrent writes to lease db file are possible
We were having networking issues where some network traffic got heavily stalled or lost, can't tell for sure at this point, but after this event we've detected corrupted lease files on ~300 of ~3000 virtual machines running on this network.
All of them looked like some form of the following:
[snip]
lease {
interface lease {
interface "eth0";
fixed-address X.X.X.X;
option subnet-mask 255.255.252.0;
option routers X.X.X.X;
option dhcp-lease-time 86400;
option dhcp-message-type 5;
option domain-name-servers 8.8.8.8,1.1.1.1,8.8.4.4;
option dhcp-server-identifier X.X.X.X;
option domain-search "";
option interface-mtu 1500;
option dhcp-renewal-time 40003;
option broadcast-address X.X.X.X;
option dhcp-rebinding-time 72403;
option host-name "host-X-X-X-X";
option domain-name "host-X-X-X-X";
renew 1 2022/08/29 10:25:22;
rebind 1 2022/08/29 21:00:55;
expire 2 202lease {
interface lease {
interface "eth0";
fixed-address X.X.X.X;
option subnet-mask 255.255.252.0;
option routers X.X.X.X;
option dhcp-lease-time 86400;
option dhcp-message-type 5;
option domain-name-servers 8.8.8.8,1.1.1.1,8.8.4.4;
option dhcp-server-identifier X.X.X.X;
option domain-search "";
option interface-mtu 1500;
option dhcp-renewal-time 39358;
option broadcast-address X.X.X.X;
option dhcp-rebinding-time 71758;
option host-name "host-X-X-X-X";
option domain-name "host-X-X-X-X";
renew 1 2022/08/29 19:10:19;
rebind 2 2022/08/30 06:21:20;
expire 2 202lease {
interface lease {
interface "eth0";
fixed-address X.X.X.X;
option subnet-mask 255.255.252.0;
option routers X.X.X.X;
option dhcp-lease-time 86400;
option dhcp-message-type 5;
option domain-name-servers 8.8.8.8,1.1.1.1,8.8.4.4;
option dhcp-server-identifier X.X.X.X;
option domain-search "";
option interface-mtu 1500;
option dhcp-renewal-time 39874;
option broadcast-address X.X.X.X;
option dhcp-rebinding-time 72274;
option host-name "host-X-X-X-X";
option domain-name "host-X-X-X-X";
renew 2 2022/08/30 05:26:46;
rebind 2 2022/08/30 15:14:53;
expire 2 202lease {
interface lease {
interface "eth0";
fixed-address X.X.X.X;
option subnet-mask 255.255.252.0;
option routers X.X.X.X;
option dhcp-lease-time 86400;
option dhcp-message-type 5;
option domain-name-servers 8.8.8.8,1.1.1.1,8.8.4.4;
option dhcp-server-identifier X.X.X.X;
option domain-search "";
option interface-mtu 1500;
option dhcp-renewal-time 39483;
option broadcast-address X.X.X.X;
option dhcp-rebinding-time 71883;
option host-name "host-X-X-X-X";
option domain-name "host-X-X-X-X";
renew 2 2022/08/30 16:20:10;
rebind 3 2022/08/31 01:24:50;
expire 3 202lease {
interface "eth0";
fixed-address X.X.X.X;
option subnet-mask 255.255.252.0;
option routers X.X.X.X;
option dhcp-lease-time 86400;
option dhcp-message-type 5;
option domain-name-servers 8.8.8.8,1.1.1.1,8.8.4.4;
option dhcp-server-identifier X.X.X.X;
option domain-search "";
option interface-mtu 1500;
option dhcp-renewal-time 39823;
option broadcast-address X.X.X.X;
option dhcp-rebinding-time 72223;
option host-name "host-X-X-X-X";
option domain-name "host-X-X-X-X";
renew 2 2022/08/30 18:17:09;
rebind 3 2022/08/31 05:48:17;
expire 3 2022/08/31 09:44:34;
}
There have been cases reported online where a lease file has corrupt data:
https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=945441
https://lists.debian.org/debian-user/2006/11/msg03263.html
Replicating the issue will be difficult because we don't know what exactly caused our networking issues yet.