diff --git a/app/playlists.py b/app/playlists.py
index 68b9c63..80a26ca 100644
--- a/app/playlists.py
+++ b/app/playlists.py
@@ -209,11 +209,7 @@ class Playlist(QTableWidget):
 
             # Add start times or empty items, otherwise background
             # colour won't be set for columns without items
-            if start_time:
-                item = QTableWidgetItem(start_time.strftime("%H:%M:%S"))
-            else:
-                item = QTableWidgetItem()
-            self.setItem(row, self.COL_START_TIME, item)
+            self.set_row_time(row, start_time)
             item = QTableWidgetItem()
             self.setItem(row, self.COL_PATH, item)
 
@@ -349,6 +345,17 @@ class Playlist(QTableWidget):
         except AttributeError:
             return ""
 
+    def get_row_time(self, row):
+        try:
+            if self.item(row, self.COL_START_TIME):
+                return datetime.strptime(self.item(
+                    row, self.COL_START_TIME).text(), "%H:%M:%S"
+                )
+            else:
+                return None
+        except ValueError:
+            return None
+
     def calculate_next_start_time(self, row, start):
         "Return this row's end time given its start time"
 
@@ -633,85 +640,72 @@ class Playlist(QTableWidget):
         # Set colours and start times
         next_start_time = None
 
-        # Set current row if there is one
-        row = current
-        if row is not None:
-            # Set start time
-            next_start_time = self.calculate_next_start_time(
-                row, self.current_track.start_time)
-            item = QTableWidgetItem(
-                self.current_track.start_time.strftime("%H:%M:%S")
-            )
-            self.setItem(row, self.COL_START_TIME, item)
-            # Set colour
-            self.set_row_colour(
-                current, QColor(Config.COLOUR_CURRENT_PLAYLIST)
-            )
-            # Make bold
-            self.set_row_bold(current)
-
-        # Now from the top, but ignore timing until either next or current
+        # Cycle through all rows
         for row in range(self.rowCount()):
-            # Current row is already set
-            if row == current:
-                continue
+            # We can't calculate start times until next_start_time is
+            # set. That can be set by either a note with a time, or the
+            # current track.
 
-            set_time = row >= min(next or 0, current or 0)
-
-            # Handle notes
             if row in notes:
-                if set_time:
-                    try:
-                        next_start_time = datetime.strptime(
-                            self.item(row, self.COL_START_TIME).text(),
-                            "%H:%M:%S"
-                        )
-                    except ValueError:
-                        pass
+                row_time = self.get_row_time(row)
+                if row_time:
+                    next_start_time = row_time
                 # Set colour
                 self.set_row_colour(
                     row, QColor(Config.COLOUR_NOTES_PLAYLIST)
                 )
                 self.set_row_bold(row)
 
-            # Handle next
-            elif row == next:
-                if set_time and next_start_time:
-                    item = QTableWidgetItem(
-                        next_start_time.strftime("%H:%M:%S")
-                    )
-                    self.setItem(row, self.COL_START_TIME, item)
-                    next_start_time = self.calculate_next_start_time(
-                        row, next_start_time)
+            elif row == current:
+                # Set start time
+                self.set_row_time(row, self.current_track.start_time)
+                # Calculate next_start_time
+                next_start_time = self.calculate_next_start_time(
+                    row, self.current_track.start_time)
                 # Set colour
-                self.set_row_colour(
-                    row, QColor(Config.COLOUR_NEXT_PLAYLIST)
-                )
+                self.set_row_colour(row, QColor(
+                    Config.COLOUR_CURRENT_PLAYLIST))
+                # Make bold
+                self.set_row_bold(row)
+
+            elif row == next:
+                # if there's a current row, set start time from that
+                if self.current_track:
+                    start_time = self.calculate_next_start_time(
+                        current, self.current_track.start_time)
+                else:
+                    # No current track to base from, but don't change
+                    # time if it's already set
+                    start_time = self.get_row_time(row)
+                    if not start_time:
+                        start_time = next_start_time
+                # Now set it
+                self.set_row_time(row, start_time)
+                next_start_time = self.calculate_next_start_time(
+                    row, start_time)
+                # Set colour
+                self.set_row_colour(row, QColor(Config.COLOUR_NEXT_PLAYLIST))
                 # Make bold
                 self.set_row_bold(row)
 
-            # Handle other rows
             else:
-                # Stripe rows
+                # Stripe remaining rows
                 if row % 2:
                     colour = QColor(Config.COLOUR_ODD_PLAYLIST)
                 else:
                     colour = QColor(Config.COLOUR_EVEN_PLAYLIST)
                 self.set_row_colour(row, colour)
 
-                # Set time
-                if set_time and next_start_time:
-                    item = QTableWidgetItem(
-                        next_start_time.strftime("%H:%M:%S"))
-                    self.setItem(row, self.COL_START_TIME, item)
-                    next_start_time = self.calculate_next_start_time(
-                        row, next_start_time)
-
-                # Dim played tracks
-                track_id = int(self.item(row, self.COL_INDEX).text())
-                if track_id in self.played_tracks:
+                if (int(self.item(row, self.COL_INDEX).text()) in
+                        self.played_tracks):
                     self.set_row_not_bold(row)
                 else:
+                    # Set time only if we haven't played it yet
+                    if next_start_time:
+                        self.set_row_time(row, next_start_time)
+                        next_start_time = self.calculate_next_start_time(
+                            row, next_start_time)
+                    # Don't dim unplayed tracks
                     self.set_row_bold(row)
 
         # Headers might need updating
@@ -843,6 +837,14 @@ class Playlist(QTableWidget):
     def set_row_not_bold(self, row):
         self.set_row_bold(row, False)
 
+    def set_row_time(self, row, time):
+        try:
+            time_str = time.strftime("%H:%M:%S")
+        except AttributeError:
+            time_str = ""
+        item = QTableWidgetItem(time_str)
+        self.setItem(row, self.COL_START_TIME, item)
+
     def tracks_changed(self):
         """
         One or more of current or next track has changed.