diff --git a/dir.c b/dir.c
index 570b651a1..aa1a7181f 100644
--- a/dir.c
+++ b/dir.c
@@ -1088,7 +1088,7 @@ int remove_dir_recursively(struct strbuf *path, int flag)
 
 	dir = opendir(path->buf);
 	if (!dir)
-		return -1;
+		return rmdir(path->buf);
 	if (path->buf[original_len - 1] != '/')
 		strbuf_addch(path, '/');
 
diff --git a/t/t7300-clean.sh b/t/t7300-clean.sh
index 02f67b73b..9b12681f9 100755
--- a/t/t7300-clean.sh
+++ b/t/t7300-clean.sh
@@ -453,4 +453,11 @@ test_expect_success 'git clean -e' '
 	)
 '
 
+test_expect_success SANITY 'git clean -d with an unreadable empty directory' '
+	mkdir foo &&
+	chmod a= foo &&
+	git clean -dfx foo &&
+	! test -d foo
+'
+
 test_done