diff --git a/player/audio.c b/player/audio.c
index 4fe78953db843..b6e811861279d 100644
--- a/player/audio.c
+++ b/player/audio.c
@@ -213,6 +213,7 @@ static void ao_chain_reset_state(struct ao_chain *ao_c)
     ao_c->start_pts = MP_NOPTS_VALUE;
     ao_c->untimed_throttle = false;
     ao_c->underrun = false;
+    ao_c->delaying_audio_start = false;
 }
 
 void reset_audio_state(struct MPContext *mpctx)
@@ -841,12 +842,13 @@ void audio_start_ao(struct MPContext *mpctx)
             MP_VERBOSE(mpctx, "delaying audio start %f vs. %f, diff=%f\n",
                        apts, pts, diff);
             mpctx->logged_async_diff = diff;
+            ao_c->delaying_audio_start = true;
         }
         return;
     }
 
     MP_VERBOSE(mpctx, "starting audio playback\n");
-    ao_c->audio_started = true;
+    ao_c->delaying_audio_start = false;
     ao_start(ao_c->ao);
     mpctx->audio_status = STATUS_PLAYING;
     if (ao_c->out_eof) {
diff --git a/player/core.h b/player/core.h
index 28acc26c5df5b..e4bdbad972d2d 100644
--- a/player/core.h
+++ b/player/core.h
@@ -194,7 +194,7 @@ struct ao_chain {
     double start_pts;
     bool start_pts_known;
 
-    bool audio_started;
+    bool delaying_audio_start;
 
     struct track *track;
     struct mp_pin *filter_src;
diff --git a/player/video.c b/player/video.c
index ece77e91aafb9..ee2da584cc8d6 100644
--- a/player/video.c
+++ b/player/video.c
@@ -388,7 +388,7 @@ static void handle_new_frame(struct MPContext *mpctx)
         }
     }
     mpctx->time_frame += frame_time / mpctx->video_speed;
-    if (mpctx->ao_chain && mpctx->ao_chain->audio_started)
+    if (mpctx->ao_chain && !mpctx->ao_chain->delaying_audio_start)
         mpctx->delay -= frame_time;
     if (mpctx->video_status >= STATUS_PLAYING)
         adjust_sync(mpctx, pts, frame_time);