Introduction
Situation: Your monitoring is warning that the root filesystem of a docker host is filling up. You realize that one or more containers are filling up their corresponding overlay2 filesystems.
root@prd-docker-l01:/var/lib/docker/overlay2# du -hs * | sort -h
912M e050850d947bab0a1bf1143d0514a65070a651fc2a01dee87711704542bb7ec1
1.1G 2e910edbec7e137655736f38185680ef8a5948c22e74c02e1bd121c9adabc265
1.3G 2e9f4aa26682df9a0e3e19ac07cdbc7eca29f506d695e8adc337409cab90809e
1.5G 80f8c9d45565aa68a75e1f8dd7f5aa0c98dc1bf3ae7a041ffcaa799889a0325d
root@prd-docker-l01:/var/lib/docker/overlay2# du -hs .
21G .
Which container’s overlay2 is growing?
docker ps -q | while read -r container_id; do overlay_dir=$(docker inspect --format='{{.GraphDriver.Data.MergedDir}}' $container_id | sed 's:/merged$::'); overlay_id=$(basename $overlay_dir); size=$(du -hs $overlay_dir | awk '{print $1}'); container_name=$(docker ps | grep $container_id | awk '{print $NF}') ; echo "$size Container: $container_name/$container_id - Overlay: $overlay_id - Size: $size"; done | sort -h
912M Container: my-container-1/cd9885b986fb - Overlay: e050850d947bab0a1bf1143d0514a65070a651fc2a01dee87711704542bb7ec1 - Size: 912M
1.1G Container: admin-gui-admin-gui-1/719714a4275a - Overlay: 2e910edbec7e137655736f38185680ef8a5948c22e74c02e1bd121c9adabc265 - Size: 1.1G
1.5G Container: some-other-container-1/4c6ec45fe606 - Overlay: 80f8c9d45565aa68a75e1f8dd7f5aa0c98dc1bf3ae7a041ffcaa799889a0325d - Size: 1.5G
Bonus: Find overlay2 directories that can be deleted (not belonging to running containers). Do not delete the l directory.
find /var/lib/docker/overlay2 -mindepth 1 -maxdepth 1 -type d \! -exec bash -c "docker ps -q | xargs docker inspect --format '{{.GraphDriver.Data}}'| grep -q "{} \; -print