diff --git a/services/surfaceflinger/Scheduler/LayerInfo.cpp b/services/surfaceflinger/Scheduler/LayerInfo.cpp
index 348e2b9..875e870 100644
--- a/services/surfaceflinger/Scheduler/LayerInfo.cpp
+++ b/services/surfaceflinger/Scheduler/LayerInfo.cpp
@@ -114,12 +114,24 @@
         }
     }
 
+    // Vote the small dirty when a layer contains at least HISTORY_SIZE of small dirty updates.
+    bool isSmallDirty = false;
+    if (smallDirtyCount >= kNumSmallDirtyThreshold) {
+        if (mLastSmallDirtyCount >= HISTORY_SIZE) {
+            isSmallDirty = true;
+        } else {
+            mLastSmallDirtyCount++;
+        }
+    } else {
+        mLastSmallDirtyCount = 0;
+    }
+
     if (isFrequent || isInfrequent) {
         // If the layer was previously inconclusive, we clear
         // the history as indeterminate layers changed to frequent,
         // and we should not look at the stale data.
         return {isFrequent, isFrequent && !mIsFrequencyConclusive, /* isConclusive */ true,
-                /* isSmallDirty */ smallDirtyCount >= kNumSmallDirtyThreshold};
+                isSmallDirty};
     }
 
     // If we can't determine whether the layer is frequent or not, we return
@@ -304,6 +316,7 @@
         ATRACE_FORMAT_INSTANT("infrequent");
         ALOGV("%s is infrequent", mName.c_str());
         mLastRefreshRate.infrequent = true;
+        mLastSmallDirtyCount = 0;
         // Infrequent layers vote for minimal refresh rate for
         // battery saving purposes and also to prevent b/135718869.
         return {LayerHistory::LayerVoteType::Min, Fps()};
@@ -313,7 +326,7 @@
         clearHistory(now);
     }
 
-    // Return no vote if the latest frames are small dirty.
+    // Return no vote if the recent frames are small dirty.
     if (frequent.isSmallDirty && !mLastRefreshRate.reported.isValid()) {
         ATRACE_FORMAT_INSTANT("NoVote (small dirty)");
         ALOGV("%s is small dirty", mName.c_str());
diff --git a/services/surfaceflinger/Scheduler/LayerInfo.h b/services/surfaceflinger/Scheduler/LayerInfo.h
index 6a85806..122796b 100644
--- a/services/surfaceflinger/Scheduler/LayerInfo.h
+++ b/services/surfaceflinger/Scheduler/LayerInfo.h
@@ -306,6 +306,10 @@
 
     RefreshRateHistory mRefreshRateHistory;
 
+    // This will be accessed from only one thread when counting a layer is frequent or infrequent,
+    // and to determine whether a layer is in small dirty updating.
+    mutable int32_t mLastSmallDirtyCount = 0;
+
     mutable std::unordered_map<LayerHistory::LayerVoteType, std::string> mTraceTags;
 
     // Shared for all LayerInfo instances
diff --git a/services/surfaceflinger/tests/unittests/LayerHistoryTest.cpp b/services/surfaceflinger/tests/unittests/LayerHistoryTest.cpp
index be4b026..69128c0 100644
--- a/services/surfaceflinger/tests/unittests/LayerHistoryTest.cpp
+++ b/services/surfaceflinger/tests/unittests/LayerHistoryTest.cpp
@@ -1013,8 +1013,8 @@
 
     LayerHistory::Summary summary;
 
-    // layer1 is active but infrequent.
-    for (int i = 0; i < PRESENT_TIME_HISTORY_SIZE; i++) {
+    // layer1 is updating small dirty.
+    for (int i = 0; i < PRESENT_TIME_HISTORY_SIZE + FREQUENT_LAYER_WINDOW_SIZE + 1; i++) {
         auto props = layer1->getLayerProps();
         props.isSmallDirty = true;
         history().record(layer1->getSequence(), props, 0 /*presentTime*/, time,