diff --git a/cmd/minikube/cmd/delete.go b/cmd/minikube/cmd/delete.go
index dd34cd4d53e2..639710e6f98b 100644
--- a/cmd/minikube/cmd/delete.go
+++ b/cmd/minikube/cmd/delete.go
@@ -757,24 +757,31 @@ func deleteKnownHosts() {
 	}
 	for _, file := range fileInfo {
 		if file.IsDir() {
-			nodeName := file.Name()
-
-			knownHostPath := filepath.Join(localpath.MiniPath(), "machines", nodeName, "known_host")
-			if _, err := os.Stat(knownHostPath); err == nil {
-				// if this file exists, remove this line from known_hosts
-				key, err := os.ReadFile(knownHostPath)
-				if err != nil {
-					klog.Warningf("error reading keys from %s: %v", knownHostPath, err)
-					continue
-				}
-				if err := util.RemoveLineFromFile(string(key), knownHosts); err != nil {
-					klog.Warningf("failed to remove key: %v", err)
-				}
-				// and, remove the file which stores this key
-				if err := os.Remove(knownHostPath); err != nil {
-					klog.Warningf("failed to remove key: %v", err)
-				}
-			}
+			continue
+		}
+		nodeName := file.Name()
+
+		knownHostPath := filepath.Join(localpath.MiniPath(), "machines", nodeName, "known_host")
+		_, err := os.Stat(knownHostPath)
+		if errors.Is(err, os.ErrNotExist) {
+			continue
+		}
+		if err != nil {
+			klog.Warningf("error reading %s: %v", knownHostPath, err)
+			continue
+		}
+		// if this file exists, remove this line from known_hosts
+		key, err := os.ReadFile(knownHostPath)
+		if err != nil {
+			klog.Warningf("error reading key from %s: %v", knownHostPath, err)
+			continue
+		}
+		if err := util.RemoveLineFromFile(string(key), knownHosts); err != nil {
+			klog.Warningf("failed to remove key: %v", err)
+		}
+		// and, remove the file which stores this key
+		if err := os.Remove(knownHostPath); err != nil {
+			klog.Warningf("failed to remove %s: %v", knownHostPath, err)
 		}
 	}
 
diff --git a/cmd/minikube/cmd/ssh-host.go b/cmd/minikube/cmd/ssh-host.go
index f524523c57a5..cff84e830b25 100644
--- a/cmd/minikube/cmd/ssh-host.go
+++ b/cmd/minikube/cmd/ssh-host.go
@@ -130,7 +130,7 @@ func appendKnownHelper(nodeName string, appendKnown bool) error {
 		// these keys can be removed properly
 		_, cc := mustload.Partial(ClusterFlagValue())
 		knownHostPath := filepath.Join(localpath.MiniPath(), "machines", config.MachineName(*cc, *n), "known_host")
-		if err := os.WriteFile(knownHostPath, []byte(keys), 0666); err != nil {
+		if err := os.WriteFile(knownHostPath, []byte(keys), 0644); err != nil {
 			return fmt.Errorf("WriteFile to %s: %v", knownHostPath, err)
 		}
 	}
diff --git a/pkg/util/utils_test.go b/pkg/util/utils_test.go
index ceacd3008f7d..6ff22336b7f3 100644
--- a/pkg/util/utils_test.go
+++ b/pkg/util/utils_test.go
@@ -324,19 +324,19 @@ func TestRemoveLineFromFile(t *testing.T) {
 	filePath := "./test_known_hosts"
 	// first we write the content into the test file
 	if err := os.WriteFile(filePath, []byte(testFileContent), 0666); err != nil {
-		t.Errorf("Failed to write to test file, err=%v", err)
+		t.Fatalf("Failed to write to test file, err=%v", err)
 	}
 	defer os.Remove(filePath)
 
 	line := "35.244.161.222  remove.test"
 	if err := RemoveLineFromFile(line, filePath); err != nil {
-		t.Errorf("Failed to run RemoveLineFromFile, err=%v", err)
+		t.Fatalf("Failed to run RemoveLineFromFile, err=%v", err)
 	}
 	// now read the file again to see whether this line still exists
 
 	newContent, err := os.ReadFile(filePath)
 	if err != nil {
-		t.Errorf("Failed to read file, err=%v", err)
+		t.Fatalf("Failed to read file, err=%v", err)
 	}
 	if strings.Contains(string(newContent), line) {
 		t.Errorf("Failed to remove the line")