diff --git a/keystore2/src/async_task.rs b/keystore2/src/async_task.rs
index e130024..0515c8f 100644
--- a/keystore2/src/async_task.rs
+++ b/keystore2/src/async_task.rs
@@ -417,7 +417,9 @@
             Err(RecvTimeoutError::Timeout)
         );
         done_receiver.recv().unwrap();
-        idle_done_receiver.recv_timeout(Duration::from_millis(1)).unwrap();
+        // Now that the last low-priority job has completed, the idle task should
+        // fire pretty much immediately.
+        idle_done_receiver.recv_timeout(Duration::from_millis(50)).unwrap();
 
         // Idle callback not executed again even if we wait for a while.
         assert_eq!(
@@ -438,7 +440,7 @@
             Err(RecvTimeoutError::Timeout)
         );
         done_receiver.recv().unwrap();
-        idle_done_receiver.recv_timeout(Duration::from_millis(1)).unwrap();
+        idle_done_receiver.recv_timeout(Duration::from_millis(50)).unwrap();
     }
 
     #[test]