diff --git a/modules/services/gpg-agent.nix b/modules/services/gpg-agent.nix
index 9d176047c..c7fefae62 100644
--- a/modules/services/gpg-agent.nix
+++ b/modules/services/gpg-agent.nix
@@ -371,7 +371,8 @@ in
         );
 
         home.sessionVariablesExtra = optionalString cfg.enableSshSupport ''
-          if [ -z "$SSH_AUTH_SOCK" ]; then
+          unset SSH_AGENT_PID
+          if [ "''${gnupg_SSH_AUTH_SOCK_by:-0}" -ne $$ ]; then
             export SSH_AUTH_SOCK="$(${gpgPkg}/bin/gpgconf --list-dirs agent-ssh-socket)"
           fi
         '';