From 2827df88e441e480ad657bc9942f7806f7575f9f Mon Sep 17 00:00:00 2001
From: Frank Bossen <fbossen@gmail.com>
Date: Mon, 19 Apr 2021 15:15:36 -0400
Subject: [PATCH] Fix #1449: output only layers in active OLS

---
 source/Lib/DecoderLib/DecLib.cpp | 56 ++++++++++++++++++++++++++++++++
 1 file changed, 56 insertions(+)

diff --git a/source/Lib/DecoderLib/DecLib.cpp b/source/Lib/DecoderLib/DecLib.cpp
index ed881e8c0a..f6bd5008ba 100644
--- a/source/Lib/DecoderLib/DecLib.cpp
+++ b/source/Lib/DecoderLib/DecLib.cpp
@@ -690,6 +690,34 @@ void DecLib::finishPictureLight(int& poc, PicList*& rpcListPic )
   Slice*  pcSlice = m_pcPic->cs->slice;
 
   m_pcPic->neededForOutput = (pcSlice->getPicHeader()->getPicOutputFlag() ? true : false);
+
+  const VPS *vps = pcSlice->getVPS();
+  if (vps != nullptr)
+  {
+    if (!vps->getEachLayerIsAnOlsFlag())
+    {
+      const int layerId        = pcSlice->getNalUnitLayerId();
+      const int generalLayerId = vps->getGeneralLayerIdx(layerId);
+      bool      layerIsOutput  = true;
+
+      if (vps->getOlsModeIdc() == 0)
+      {
+        layerIsOutput = generalLayerId == vps->m_targetOlsIdx;
+      }
+      else if (vps->getOlsModeIdc() == 1)
+      {
+        layerIsOutput = generalLayerId <= vps->m_targetOlsIdx;
+      }
+      else if (vps->getOlsModeIdc() == 2)
+      {
+        layerIsOutput = vps->getOlsOutputLayerFlag(vps->m_targetOlsIdx, generalLayerId);
+      }
+      if (!layerIsOutput)
+      {
+        m_pcPic->neededForOutput = false;
+      }
+    }
+  }
   m_pcPic->reconstructed = true;
 
   Slice::sortPicList( m_cListPic ); // sorting for application output
@@ -822,6 +850,34 @@ void DecLib::finishPicture(int &poc, PicList *&rpcListPic, MsgLevel msgl, bool a
       }
     }
   }
+
+  const VPS *vps = pcSlice->getVPS();
+  if (vps != nullptr)
+  {
+    if (!vps->getEachLayerIsAnOlsFlag())
+    {
+      const int layerId        = pcSlice->getNalUnitLayerId();
+      const int generalLayerId = vps->getGeneralLayerIdx(layerId);
+      bool      layerIsOutput  = true;
+
+      if (vps->getOlsModeIdc() == 0)
+      {
+        layerIsOutput = generalLayerId == vps->m_targetOlsIdx;
+      }
+      else if (vps->getOlsModeIdc() == 1)
+      {
+        layerIsOutput = generalLayerId <= vps->m_targetOlsIdx;
+      }
+      else if (vps->getOlsModeIdc() == 2)
+      {
+        layerIsOutput = vps->getOlsOutputLayerFlag(vps->m_targetOlsIdx, generalLayerId);
+      }
+      if (!layerIsOutput)
+      {
+        m_pcPic->neededForOutput = false;
+      }
+    }
+  }
   m_pcPic->reconstructed = true;
 
   // process buffered suffix APS NALUs
-- 
GitLab