diff --git a/services/surfaceflinger/Scheduler/LayerInfoV2.cpp b/services/surfaceflinger/Scheduler/LayerInfoV2.cpp
index f309d4d..345b8f9 100644
--- a/services/surfaceflinger/Scheduler/LayerInfoV2.cpp
+++ b/services/surfaceflinger/Scheduler/LayerInfoV2.cpp
@@ -54,21 +54,40 @@
     return mFrameTimes.back().queueTime >= getActiveLayerThreshold(now);
 }
 
+bool LayerInfoV2::isFrameTimeValid(const FrameTimeData& frameTime) const {
+    return frameTime.queueTime >= std::chrono::duration_cast<std::chrono::nanoseconds>(
+                                          mFrameTimeValidSince.time_since_epoch())
+                                          .count();
+}
+
 bool LayerInfoV2::isFrequent(nsecs_t now) const {
-    // Assume layer is infrequent if too few present times have been recorded.
+    // If we know nothing about this layer we consider it as frequent as it might be the start
+    // of an animation.
     if (mFrameTimes.size() < FREQUENT_LAYER_WINDOW_SIZE) {
-        return false;
+        return true;
     }
 
     // Layer is frequent if the earliest value in the window of most recent present times is
     // within threshold.
     const auto it = mFrameTimes.end() - FREQUENT_LAYER_WINDOW_SIZE;
+    if (!isFrameTimeValid(*it)) {
+        return true;
+    }
+
     const nsecs_t threshold = now - MAX_FREQUENT_LAYER_PERIOD_NS.count();
     return it->queueTime >= threshold;
 }
 
 bool LayerInfoV2::hasEnoughDataForHeuristic() const {
     // The layer had to publish at least HISTORY_SIZE or HISTORY_TIME of updates
+    if (mFrameTimes.size() < 2) {
+        return false;
+    }
+
+    if (!isFrameTimeValid(mFrameTimes.front())) {
+        return false;
+    }
+
     if (mFrameTimes.size() < HISTORY_SIZE &&
         mFrameTimes.back().queueTime - mFrameTimes.front().queueTime < HISTORY_TIME.count()) {
         return false;
diff --git a/services/surfaceflinger/Scheduler/LayerInfoV2.h b/services/surfaceflinger/Scheduler/LayerInfoV2.h
index 564f05e..90f6310 100644
--- a/services/surfaceflinger/Scheduler/LayerInfoV2.h
+++ b/services/surfaceflinger/Scheduler/LayerInfoV2.h
@@ -82,12 +82,25 @@
     // updated time, the updated time is the present time.
     nsecs_t getLastUpdatedTime() const { return mLastUpdatedTime; }
 
-    void clearHistory() { mFrameTimes.clear(); }
+    void clearHistory() {
+        // Mark mFrameTimeValidSince to now to ignore all previous frame times.
+        // We are not deleting the old frame to keep track of whether we should treat the first
+        // buffer as Max as we don't know anything about this layer or Min as this layer is
+        // posting infrequent updates.
+        mFrameTimeValidSince = std::chrono::steady_clock::now();
+    }
 
 private:
+    // Used to store the layer timestamps
+    struct FrameTimeData {
+        nsecs_t presetTime; // desiredPresentTime, if provided
+        nsecs_t queueTime;  // buffer queue time
+    };
+
     bool isFrequent(nsecs_t now) const;
     bool hasEnoughDataForHeuristic() const;
     std::optional<float> calculateRefreshRateIfPossible();
+    bool isFrameTimeValid(const FrameTimeData&) const;
 
     // Used for sanitizing the heuristic data
     const nsecs_t mHighRefreshRatePeriod;
@@ -103,12 +116,9 @@
         float fps;
     } mLayerVote;
 
-    // Used to store the layer timestamps
-    struct FrameTimeData {
-        nsecs_t presetTime; // desiredPresentTime, if provided
-        nsecs_t queueTime;  // buffer queue time
-    };
     std::deque<FrameTimeData> mFrameTimes;
+    std::chrono::time_point<std::chrono::steady_clock> mFrameTimeValidSince =
+            std::chrono::steady_clock::now();
     static constexpr size_t HISTORY_SIZE = 90;
     static constexpr std::chrono::nanoseconds HISTORY_TIME = 1s;
 };
diff --git a/services/surfaceflinger/tests/unittests/LayerHistoryTestV2.cpp b/services/surfaceflinger/tests/unittests/LayerHistoryTestV2.cpp
index 922966a..91f9751 100644
--- a/services/surfaceflinger/tests/unittests/LayerHistoryTestV2.cpp
+++ b/services/surfaceflinger/tests/unittests/LayerHistoryTestV2.cpp
@@ -34,7 +34,6 @@
 
 class LayerHistoryTestV2 : public testing::Test {
 protected:
-    static constexpr auto FREQUENT_LAYER_WINDOW_SIZE = LayerInfoV2::FREQUENT_LAYER_WINDOW_SIZE;
     static constexpr auto PRESENT_TIME_HISTORY_SIZE = LayerInfoV2::HISTORY_SIZE;
     static constexpr auto MAX_FREQUENT_LAYER_PERIOD_NS = LayerInfoV2::MAX_FREQUENT_LAYER_PERIOD_NS;
 
@@ -85,7 +84,6 @@
     TestableScheduler* const mScheduler{new TestableScheduler(mConfigs, true)};
     TestableSurfaceFlinger mFlinger;
 
-    const nsecs_t mTime = systemTime();
 };
 
 namespace {
@@ -98,26 +96,25 @@
     EXPECT_EQ(1, layerCount());
     EXPECT_EQ(0, activeLayerCount());
 
+    const nsecs_t time = systemTime();
+
     // No layers returned if no layers are active.
-    EXPECT_TRUE(history().summarize(mTime).empty());
+    EXPECT_TRUE(history().summarize(time).empty());
     EXPECT_EQ(0, activeLayerCount());
 
     // Max returned if active layers have insufficient history.
     for (int i = 0; i < PRESENT_TIME_HISTORY_SIZE - 1; i++) {
-        history().record(layer.get(), 0, mTime);
-        ASSERT_EQ(1, history().summarize(mTime).size());
-        const auto expectedType = (i + 1 < FREQUENT_LAYER_WINDOW_SIZE)
-                ? LayerHistory::LayerVoteType::Min
-                : LayerHistory::LayerVoteType::Max;
-        EXPECT_EQ(expectedType, history().summarize(mTime)[0].vote);
+        history().record(layer.get(), 0, time);
+        ASSERT_EQ(1, history().summarize(time).size());
+        EXPECT_EQ(LayerHistory::LayerVoteType::Max, history().summarize(time)[0].vote);
         EXPECT_EQ(1, activeLayerCount());
     }
 
     // Max is returned since we have enough history but there is no timestamp votes.
     for (int i = 0; i < 10; i++) {
-        history().record(layer.get(), 0, mTime);
-        ASSERT_EQ(1, history().summarize(mTime).size());
-        EXPECT_EQ(LayerHistory::LayerVoteType::Max, history().summarize(mTime)[0].vote);
+        history().record(layer.get(), 0, time);
+        ASSERT_EQ(1, history().summarize(time).size());
+        EXPECT_EQ(LayerHistory::LayerVoteType::Max, history().summarize(time)[0].vote);
         EXPECT_EQ(1, activeLayerCount());
     }
 }
@@ -130,17 +127,19 @@
     EXPECT_EQ(1, layerCount());
     EXPECT_EQ(0, activeLayerCount());
 
-    history().record(layer.get(), 0, mTime);
-    auto summary = history().summarize(mTime);
-    ASSERT_EQ(1, history().summarize(mTime).size());
+    nsecs_t time = systemTime();
+
+    history().record(layer.get(), 0, time);
+    auto summary = history().summarize(time);
+    ASSERT_EQ(1, history().summarize(time).size());
     // Layer is still considered inactive so we expect to get Min
-    EXPECT_EQ(LayerHistory::LayerVoteType::Min, history().summarize(mTime)[0].vote);
+    EXPECT_EQ(LayerHistory::LayerVoteType::Max, history().summarize(time)[0].vote);
     EXPECT_EQ(1, activeLayerCount());
 
     EXPECT_CALL(*layer, isVisible()).WillRepeatedly(Return(false));
 
-    summary = history().summarize(mTime);
-    EXPECT_TRUE(history().summarize(mTime).empty());
+    summary = history().summarize(time);
+    EXPECT_TRUE(history().summarize(time).empty());
     EXPECT_EQ(0, activeLayerCount());
 }
 
@@ -152,7 +151,7 @@
     EXPECT_EQ(1, layerCount());
     EXPECT_EQ(0, activeLayerCount());
 
-    nsecs_t time = mTime;
+    nsecs_t time = systemTime();
     for (int i = 0; i < PRESENT_TIME_HISTORY_SIZE; i++) {
         history().record(layer.get(), time, time);
         time += LO_FPS_PERIOD;
@@ -175,7 +174,7 @@
     EXPECT_EQ(1, layerCount());
     EXPECT_EQ(0, activeLayerCount());
 
-    nsecs_t time = mTime;
+    nsecs_t time = systemTime();
     for (int i = 0; i < PRESENT_TIME_HISTORY_SIZE; i++) {
         history().record(layer.get(), time, time);
         time += HI_FPS_PERIOD;
@@ -202,7 +201,7 @@
     EXPECT_EQ(1, layerCount());
     EXPECT_EQ(0, activeLayerCount());
 
-    nsecs_t time = mTime;
+    nsecs_t time = systemTime();
     for (int i = 0; i < PRESENT_TIME_HISTORY_SIZE; i++) {
         history().record(layer.get(), time, time);
         time += HI_FPS_PERIOD;
@@ -230,7 +229,7 @@
     EXPECT_EQ(1, layerCount());
     EXPECT_EQ(0, activeLayerCount());
 
-    nsecs_t time = mTime;
+    nsecs_t time = systemTime();
     for (int i = 0; i < PRESENT_TIME_HISTORY_SIZE; i++) {
         history().record(layer.get(), time, time);
         time += LO_FPS_PERIOD;
@@ -256,7 +255,7 @@
     EXPECT_EQ(1, layerCount());
     EXPECT_EQ(0, activeLayerCount());
 
-    nsecs_t time = mTime;
+    nsecs_t time = systemTime();
     for (int i = 0; i < PRESENT_TIME_HISTORY_SIZE; i++) {
         history().record(layer.get(), time, time);
         time += HI_FPS_PERIOD;
@@ -291,7 +290,7 @@
     EXPECT_CALL(*layer3, isVisible()).WillRepeatedly(Return(true));
     EXPECT_CALL(*layer3, getFrameRate()).WillRepeatedly(Return(std::nullopt));
 
-    nsecs_t time = mTime;
+    nsecs_t time = systemTime();
 
     EXPECT_EQ(3, layerCount());
     EXPECT_EQ(0, activeLayerCount());
@@ -319,7 +318,7 @@
 
     ASSERT_EQ(2, history().summarize(time).size());
     EXPECT_EQ(LayerHistory::LayerVoteType::Min, history().summarize(time)[0].vote);
-    EXPECT_EQ(LayerHistory::LayerVoteType::Heuristic, history().summarize(time)[1].vote);
+    ASSERT_EQ(LayerHistory::LayerVoteType::Heuristic, history().summarize(time)[1].vote);
     EXPECT_FLOAT_EQ(HI_FPS, history().summarize(time)[1].desiredRefreshRate);
     EXPECT_EQ(2, activeLayerCount());
     EXPECT_EQ(1, frequentLayerCount(time));