diff --git a/src/sss_client/pam_sss.c b/src/sss_client/pam_sss.c
index 4590fb008d..957213afb5 100644
--- a/src/sss_client/pam_sss.c
+++ b/src/sss_client/pam_sss.c
@@ -2549,6 +2549,11 @@ static int get_authtok_for_authentication(pam_handle_t *pamh,
                 /* Fallback to password auth if no PIN was entered */
                 if (ret == EIO) {
                     ret = prompt_password(pamh, pi, _("Password: "));
+                    if (pi->pam_authtok_size == 0) {
+                        D(("Empty password failure"));
+                        pi->passkey_prompt_pin = NULL;
+                        return PAM_AUTHTOK_ERR;
+                    }
                 }
             } else {
                 ret = prompt_password(pamh, pi, _("Password: "));