diff --git a/muxer_part.go b/muxer_part.go
index f7d5d54..dfc65ea 100644
--- a/muxer_part.go
+++ b/muxer_part.go
@@ -46,7 +46,6 @@ func (p *muxerPart) finalize(endDTS time.Duration) error {
 				Samples:  track.fmp4Samples,
 			})
 
-			track.fmp4StartDTSFilled = false
 			track.fmp4Samples = nil
 		}
 	}
@@ -62,8 +61,7 @@ func (p *muxerPart) finalize(endDTS time.Duration) error {
 }
 
 func (p *muxerPart) writeSample(track *muxerTrack, sample *fmp4AugmentedSample) {
-	if !track.fmp4StartDTSFilled {
-		track.fmp4StartDTSFilled = true
+	if track.fmp4Samples == nil {
 		track.fmp4StartDTS = sample.dts
 	}
 
diff --git a/muxer_track.go b/muxer_track.go
index 048589c..55f4e75 100644
--- a/muxer_track.go
+++ b/muxer_track.go
@@ -23,7 +23,6 @@ type muxerTrack struct {
 	fmp4TimeScale             uint32               // fmp4 only
 	fmp4NextSample            *fmp4AugmentedSample // fmp4 only
 	fmp4Samples               []*fmp4.PartSample   // fmp4 only
-	fmp4StartDTSFilled        bool                 // fmp4 only
 	fmp4StartDTS              time.Duration        // fmp4 only
 }