diff --git a/modules/services/gpg-agent.nix b/modules/services/gpg-agent.nix
index c7fefae62..71b6a67fb 100644
--- a/modules/services/gpg-agent.nix
+++ b/modules/services/gpg-agent.nix
@@ -372,7 +372,7 @@ in
 
         home.sessionVariablesExtra = optionalString cfg.enableSshSupport ''
           unset SSH_AGENT_PID
-          if [ "''${gnupg_SSH_AUTH_SOCK_by:-0}" -ne $$ ]; then
+          if [ -z "$SSH_CONNECTION" -o -z "$SSH_AUTH_SOCK" ] && [ "''${gnupg_SSH_AUTH_SOCK_by:-0}" -ne $$ ]; then
             export SSH_AUTH_SOCK="$(${gpgPkg}/bin/gpgconf --list-dirs agent-ssh-socket)"
           fi
         '';