diff --git a/src/audio/pulseaudio/SDL_pulseaudio.c b/src/audio/pulseaudio/SDL_pulseaudio.c
index b5479861a..cf3522b1b 100644
--- a/src/audio/pulseaudio/SDL_pulseaudio.c
+++ b/src/audio/pulseaudio/SDL_pulseaudio.c
@@ -505,6 +505,7 @@ PULSEAUDIO_Init(SDL_AudioDriverImpl * impl)
     }
 
     if (!CheckPulseAudioAvailable()) {
+        UnloadPulseAudioLibrary();
         return 0;
     }