diff --git a/src/com/android/customization/module/ThemePickerInjector.kt b/src/com/android/customization/module/ThemePickerInjector.kt
index f31af44..a1fc014 100644
--- a/src/com/android/customization/module/ThemePickerInjector.kt
+++ b/src/com/android/customization/module/ThemePickerInjector.kt
@@ -15,6 +15,7 @@
  */
 package com.android.customization.module
 
+import android.app.Activity
 import android.app.UiModeManager
 import android.app.WallpaperManager
 import android.content.Context
@@ -111,7 +112,7 @@
     private var clockPickerInteractor: ClockPickerInteractor? = null
     private var clockSectionViewModel: ClockSectionViewModel? = null
     private var clockCarouselViewModelFactory: ClockCarouselViewModel.Factory? = null
-    private var clockViewFactories: MutableMap<Int, ClockViewFactory> = HashMap()
+    private var clockViewFactory: ClockViewFactory? = null
     private var clockPickerSnapshotRestorer: ClockPickerSnapshotRestorer? = null
     private var notificationsInteractor: NotificationsInteractor? = null
     private var notificationSectionViewModelFactory: NotificationSectionViewModel.Factory? = null
@@ -388,8 +389,7 @@
     }
 
     override fun getClockViewFactory(activity: ComponentActivity): ClockViewFactory {
-        val activityHashCode = activity.hashCode()
-        return clockViewFactories[activityHashCode]
+        return clockViewFactory
             ?: ClockViewFactory(
                     activity.applicationContext,
                     ScreenSizeCalculator.getInstance()
@@ -398,13 +398,13 @@
                     getClockRegistry(activity.applicationContext),
                 )
                 .also {
-                    clockViewFactories[activityHashCode] = it
+                    clockViewFactory = it
                     activity.lifecycle.addObserver(
                         object : DefaultLifecycleObserver {
                             override fun onDestroy(owner: LifecycleOwner) {
                                 super.onDestroy(owner)
-                                clockViewFactories[activityHashCode]?.onDestroy()
-                                clockViewFactories.remove(activityHashCode)
+                                if ((owner as Activity).isChangingConfigurations()) return
+                                clockViewFactory?.onDestroy()
                             }
                         }
                     )