diff --git a/web/src/lib/components/shared-components/scrollbar/scrollbar.svelte b/web/src/lib/components/shared-components/scrollbar/scrollbar.svelte
index 580277fba9..7dfd34e8f6 100644
--- a/web/src/lib/components/shared-components/scrollbar/scrollbar.svelte
+++ b/web/src/lib/components/shared-components/scrollbar/scrollbar.svelte
@@ -1,5 +1,6 @@
 <script lang="ts">
-  import type { AssetStore } from '$lib/stores/assets.store';
+  import type { AssetStore, AssetBucket } from '$lib/stores/assets.store';
+  import type { DateTime } from 'luxon';
   import { fromLocalDateTime } from '$lib/utils/timeline-util';
   import { createEventDispatcher } from 'svelte';
 
@@ -18,30 +19,41 @@
   const toTimelineY = (scrollY: number) => Math.round((scrollY * $assetStore.timelineHeight) / height);
 
   const HOVER_DATE_HEIGHT = 30;
+  const MIN_YEAR_LABEL_DISTANCE = 16;
 
   $: hoverY = height - windowHeight + clientY;
   $: scrollY = toScrollY(timelineY);
 
   class Segment {
-    public count;
-    public height;
-    public timeGroup;
-
-    constructor({ count = 0, height = 0, timeGroup = '' }) {
-      this.count = count;
-      this.height = height;
-      this.timeGroup = timeGroup;
-    }
+    public count = 0;
+    public height = 0;
+    public timeGroup = '';
+    public date!: DateTime;
+    public hasLabel = false;
   }
 
-  $: segments = $assetStore.buckets.map(
-    (bucket) =>
-      new Segment({
-        count: bucket.assets.length,
-        height: toScrollY(bucket.bucketHeight),
-        timeGroup: bucket.bucketDate,
-      }),
-  );
+  const calculateSegments = (buckets: AssetBucket[]) => {
+    let height = 0;
+    let prev: Segment;
+    return buckets.map((bucket) => {
+      const segment = new Segment();
+      segment.count = bucket.assets.length;
+      segment.height = toScrollY(bucket.bucketHeight);
+      segment.timeGroup = bucket.bucketDate;
+      segment.date = fromLocalDateTime(segment.timeGroup);
+
+      if (prev?.date.year !== segment.date.year && (!prev || height > MIN_YEAR_LABEL_DISTANCE)) {
+        segment.hasLabel = true;
+        height = 0;
+      }
+
+      height += segment.height;
+      prev = segment;
+      return segment;
+    });
+  };
+
+  $: segments = calculateSegments($assetStore.buckets);
 
   const dispatch = createEventDispatcher<{ scrollTimeline: number }>();
   const scrollTimeline = () => dispatch('scrollTimeline', toTimelineY(hoverY));
@@ -67,23 +79,6 @@
       isAnimating = false;
     });
   };
-
-  const prevYearSegmentHeight = (segments: Segment[], index: number) => {
-    var prevYear = null;
-    var height = 0;
-    for (var i = index; i >= 0; i--) {
-      const curr = segments[i];
-      const currYear = fromLocalDateTime(curr.timeGroup).year;
-      if (prevYear && prevYear != currYear) {
-        break;
-      }
-
-      height += curr.height;
-      prevYear = currYear;
-    }
-
-    return height;
-  };
 </script>
 
 <svelte:window bind:innerHeight={windowHeight} />
@@ -125,11 +120,8 @@
       />
     {/if}
     <!-- Time Segment -->
-    {#each segments as segment, index (segment.timeGroup)}
-      {@const date = fromLocalDateTime(segment.timeGroup)}
-      {@const year = date.year}
-      {@const label = `${date.toLocaleString({ month: 'short' })} ${year}`}
-      {@const lastGroupYear = fromLocalDateTime(segments[index - 1]?.timeGroup).year}
+    {#each segments as segment}
+      {@const label = `${segment.date.toLocaleString({ month: 'short' })} ${segment.date.year}`}
 
       <!-- svelte-ignore a11y-no-static-element-interactions -->
       <div
@@ -139,12 +131,12 @@
         aria-label={segment.timeGroup + ' ' + segment.count}
         on:mousemove={() => (hoverLabel = label)}
       >
-        {#if lastGroupYear !== year && (index == 0 || prevYearSegmentHeight(segments, index - 1) > 16)}
+        {#if segment.hasLabel}
           <div
             aria-label={segment.timeGroup + ' ' + segment.count}
             class="absolute right-0 z-10 pr-5 text-xs font-medium dark:text-immich-dark-fg font-mono"
           >
-            {year}
+            {segment.date.year}
           </div>
         {:else if segment.height > 5}
           <div