From b7939e5f97d5acf7537a30a2de3277d696fab2ab Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Thu, 25 Mar 2021 09:14:46 +0000 Subject: [PATCH] VR: fix rsyslog compresses log files but not release disk space in VR We had critical issue with VR recently. The VRs of shared network or vpc stops working after some days. After investigation, I found that the disk space is full ``` root@r-10-VM:~# df Filesystem 1K-blocks Used Available Use% Mounted on /dev/vda5 2086316 2069932 0 100% / ``` logrotate/ryslog has compresses the log files, but space is not released. see `lsof |grep deleted` ``` root@r-10-VM:~# lsof |grep deleted rsyslogd 960 root 12w REG 254,5 493060096 137 /var/log/daemon.log.1 (deleted) rsyslogd 960 root 13w REG 254,5 17715200 110 /var/log/messages.1 (deleted) rsyslogd 960 root 16w REG 254,5 545968128 342 /var/log/auth.log.1 (deleted) rsyslogd 960 root 18w REG 254,5 38313984 341 /var/log/cron.log.1 (deleted) rsyslogd 960 962 in:imuxso root 12w REG 254,5 493060096 137 /var/log/daemon.log.1 (deleted) rsyslogd 960 962 in:imuxso root 13w REG 254,5 17715200 110 /var/log/messages.1 (deleted) rsyslogd 960 962 in:imuxso root 16w REG 254,5 545968128 342 /var/log/auth.log.1 (deleted) rsyslogd 960 962 in:imuxso root 18w REG 254,5 38313984 341 /var/log/cron.log.1 (deleted) rsyslogd 960 963 in:imklog root 12w REG 254,5 493060096 137 /var/log/daemon.log.1 (deleted) rsyslogd 960 963 in:imklog root 13w REG 254,5 17715200 110 /var/log/messages.1 (deleted) rsyslogd 960 963 in:imklog root 16w REG 254,5 545968128 342 /var/log/auth.log.1 (deleted) rsyslogd 960 963 in:imklog root 18w REG 254,5 38313984 341 /var/log/cron.log.1 (deleted) rsyslogd 960 964 in:imfile root 12w REG 254,5 493060096 137 /var/log/daemon.log.1 (deleted) rsyslogd 960 964 in:imfile root 13w REG 254,5 17715200 110 /var/log/messages.1 (deleted) rsyslogd 960 964 in:imfile root 16w REG 254,5 545968128 342 /var/log/auth.log.1 (deleted) rsyslogd 960 964 in:imfile root 18w REG 254,5 38313984 341 /var/log/cron.log.1 (deleted) rsyslogd 960 965 in:imudp root 12w REG 254,5 493060096 137 /var/log/daemon.log.1 (deleted) rsyslogd 960 965 in:imudp root 13w REG 254,5 17715200 110 /var/log/messages.1 (deleted) rsyslogd 960 965 in:imudp root 16w REG 254,5 545968128 342 /var/log/auth.log.1 (deleted) rsyslogd 960 965 in:imudp root 18w REG 254,5 38313984 341 /var/log/cron.log.1 (deleted) rsyslogd 960 966 rs:main root 12w REG 254,5 493060096 137 /var/log/daemon.log.1 (deleted) rsyslogd 960 966 rs:main root 13w REG 254,5 17715200 110 /var/log/messages.1 (deleted) rsyslogd 960 966 rs:main root 16w REG 254,5 545968128 342 /var/log/auth.log.1 (deleted) rsyslogd 960 966 rs:main root 18w REG 254,5 38313984 341 /var/log/cron.log.1 (deleted) ``` workaround: restarting rsyslog to release the space. ``` systemctl restart rsyslog ``` The root cause is, the following command does not work in 4.15 template ``` root@r-10-VM:~# invoke-rc.d rsyslog rotate [FAIL] Closing open files: rsyslogd failed! ``` Fix: use `/usr/lib/rsyslog/rsyslog-rotate` instead ``` root@r-10-VM:~# /usr/lib/rsyslog/rsyslog-rotate root@r-10-VM:~# cat /usr/lib/rsyslog/rsyslog-rotate if [ -d /run/systemd/system ]; then systemctl kill -s HUP rsyslog.service else invoke-rc.d rsyslog rotate > /dev/null fi ``` --- systemvm/debian/etc/logrotate.d/rsyslog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/systemvm/debian/etc/logrotate.d/rsyslog b/systemvm/debian/etc/logrotate.d/rsyslog index 9291494481e8..cef88191606e 100644 --- a/systemvm/debian/etc/logrotate.d/rsyslog +++ b/systemvm/debian/etc/logrotate.d/rsyslog @@ -6,7 +6,7 @@ notifempty compress postrotate - /usr/sbin/invoke-rc.d rsyslog rotate > /dev/null + /usr/lib/rsyslog/rsyslog-rotate endscript } @@ -30,6 +30,6 @@ compress sharedscripts postrotate - /usr/sbin/invoke-rc.d rsyslog rotate > /dev/null + /usr/lib/rsyslog/rsyslog-rotate endscript }