Store original layer size, update capsfilter when possible

* Store original width and height in layers
* When output bins receive new video caps, change capsfilter caps to
  use new resolution, without upscaling

pkg/media/output.go

@@ -24,6 +24,7 @@ import (
 	"github.com/go-gst/go-gst/gst"
 	"github.com/go-gst/go-gst/gst/app"
 	"github.com/pion/webrtc/v3/pkg/media"
+	"google.golang.org/protobuf/proto"
 
 	"github.com/livekit/ingress/pkg/errors"
 	"github.com/livekit/ingress/pkg/stats"
@@ -76,7 +77,7 @@ type AudioOutput struct {
 	codec livekit.AudioCodec
 }
 
-func NewVideoOutput(codec livekit.VideoCodec, layer *livekit.VideoLayer, outputSync *utils.TrackOutputSynchronizer, statsGatherer *stats.LocalMediaStatsGatherer) (*VideoOutput, error) {
+func NewVideoOutput(codec livekit.VideoCodec, layer *ScaledVideoLayer, outputSync *utils.TrackOutputSynchronizer, statsGatherer *stats.LocalMediaStatsGatherer) (*VideoOutput, error) {
 	e, err := newVideoOutput(codec, outputSync)
 	if err != nil {
 		return nil, err
@@ -86,7 +87,7 @@ func NewVideoOutput(codec livekit.VideoCodec, layer *livekit.VideoLayer, outputS
 
 	e.trackStatsGatherer = statsGatherer.RegisterTrackStats(fmt.Sprintf("%s.%s", stats.OutputVideo, layer.Quality.String()))
 
-	threadCount := getVideoEncoderThreadCount(layer)
+	threadCount := getVideoEncoderThreadCount(layer.VideoLayer)
 
 	e.logger.Infow("video layer", "width", layer.Width, "height", layer.Height, "threads", threadCount)
 
@@ -132,6 +133,33 @@ func NewVideoOutput(codec livekit.VideoCodec, layer *livekit.VideoLayer, outputS
 		return nil, err
 	}
 
+	queueIn.GetStaticPad("sink").AddProbe(gst.PadProbeTypeEventDownstream, func(self *gst.Pad, info *gst.PadProbeInfo) gst.PadProbeReturn {
+		ev := info.GetEvent()
+		if ev.Type() != gst.EventTypeCaps {
+			return gst.PadProbeOK
+		}
+
+		caps := ev.ParseCaps()
+		w, h, err := getResolution(caps)
+		if err != nil {
+			logger.Errorw("input queue caps failed to get resolution", err)
+		}
+
+		l := proto.Clone(layer.VideoLayer).(*livekit.VideoLayer)
+		l.Width = layer.MaxW
+		l.Height = layer.MaxH
+		applyResolutionToLayer(l, w, h)
+
+		err = inputCaps.SetProperty("caps", gst.NewCapsFromString(
+			fmt.Sprintf("video/x-raw,width=%d,height=%d", l.Width, l.Height),
+		))
+		if err != nil {
+			logger.Errorw("failed to set input capsfilter caps", err)
+		}
+
+		return gst.PadProbeOK
+	})
+
 	queueEnc, err := gst.NewElementWithName("queue", fmt.Sprintf("video_%s_enc", layer.Quality.String()))
 	if err != nil {
 		return nil, err

pkg/media/webrtc_sink.go

@@ -180,7 +180,12 @@ func (s *WebRTCSink) addVideoTrack(w, h int) ([]*Output, error) {
 			var tracks []*lksdk.LocalTrack
 			var pliHandlers []*lksdk_output.RTCPHandler
 
-			tracks, pliHandlers, err = sdkOut.AddVideoTrack(sortedLayers, putils.GetMimeTypeForVideoCodec(s.params.VideoEncodingOptions.VideoCodec))
+			layers := make([]*livekit.VideoLayer, 0, len(sortedLayers))
+			for _, l := range sortedLayers {
+				layers = append(layers, l.VideoLayer)
+			}
+
+			tracks, pliHandlers, err = sdkOut.AddVideoTrack(layers, putils.GetMimeTypeForVideoCodec(s.params.VideoEncodingOptions.VideoCodec))
 			if err != nil {
 				return
 			}
@@ -218,20 +223,6 @@ func (s *WebRTCSink) AddTrack(kind types.StreamKind, caps *gst.Caps, p *params.P
 		}
 
 		logger.Infow("source resolution parsed", "width", w, "height", h)
-		// In cases like multi-variant HLS, the initial source resolution may be very low
-		// (e.g. 320x180). Since the input capsfilter element in the output bin will maintain this
-		// resolution, upscale to the highest layer's dimensions to prevent downscaling if we
-		// get a higher resolution variant later.
-		if len(p.VideoEncodingOptions.GetLayers()) > 0 {
-			layerHigh := p.VideoEncodingOptions.GetLayers()[0]
-			lw := int(layerHigh.GetWidth())
-			lh := int(layerHigh.GetHeight())
-			if lw > w || lh > h {
-				w = lw
-				h = lh
-				logger.Infow("max layer resolution greater than source, sizing up", "width", w, "height", h)
-			}
-		}
 
 		outputs, err := s.addVideoTrack(w, h)
 		if err != nil {
@@ -293,29 +284,30 @@ func getResolution(caps *gst.Caps) (w int, h int, err error) {
 	return wObj.(int), hObj.(int), nil
 }
 
-func filterAndSortLayersByQuality(layers []*livekit.VideoLayer, sourceW, sourceH int) []*livekit.VideoLayer {
-	layersByQuality := make(map[livekit.VideoQuality]*livekit.VideoLayer)
+type ScaledVideoLayer struct {
+	*livekit.VideoLayer
+
+	MaxW uint32
+	MaxH uint32
+}
+
+func filterAndSortLayersByQuality(layers []*livekit.VideoLayer, sourceW, sourceH int) []*ScaledVideoLayer {
+	layersByQuality := make(map[livekit.VideoQuality]*ScaledVideoLayer)
 
 	for _, layer := range layers {
-		layersByQuality[layer.Quality] = layer
+		layersByQuality[layer.Quality] = &ScaledVideoLayer{VideoLayer: layer, MaxW: layer.Width, MaxH: layer.Height}
 	}
 
-	var ret []*livekit.VideoLayer
+	var ret []*ScaledVideoLayer
 	for q := livekit.VideoQuality_LOW; q <= livekit.VideoQuality_HIGH; q++ {
 		layer, ok := layersByQuality[q]
 		if !ok {
 			continue
 		}
 
-		applyResolutionToLayer(layer, sourceW, sourceH)
+		applyResolutionToLayer(layer.VideoLayer, sourceW, sourceH)
 
 		ret = append(ret, layer)
-
-		if layer.Width >= uint32(sourceW) && layer.Height >= uint32(sourceH) {
-			// Next quality layer would be duplicate of current one
-			break
-		}
-
 	}
 	return ret
 }
@@ -334,7 +326,7 @@ func applyResolutionToLayer(layer *livekit.VideoLayer, sourceW, sourceH int) {
 		w = uint32((int64(h) * int64(sourceW)) / int64(sourceH))
 	}
 
-	// Roubd up to the next even dimension
+	// Round up to the next even dimension
 	w = ((w + 1) >> 1) << 1
 	h = ((h + 1) >> 1) << 1