From 03e736a99142d9cca218cf118b6b0e1a4e27a944 Mon Sep 17 00:00:00 2001
From: Brian Heng <brian.heng@broadcom.com>
Date: Fri, 7 Jun 2019 17:44:51 -0700
Subject: [PATCH] Fix for Ticket #311 Scaling List Bugs
- Wrong scaling list sizes used.
- Scaling factor incorrect for dependent quantization.
- 32-bit variable overflow in dependent quantization.
- Transform skip only applies to luma.
- Pred matrix ID delta adjusted for non-existent 2x2 luma lists.
---
source/Lib/CommonLib/DepQuant.cpp | 13 +++++++++++--
source/Lib/CommonLib/Quant.cpp | 24 ++++++++++++++++++++++--
source/Lib/CommonLib/QuantRDOQ.cpp | 12 ++++++++++--
source/Lib/DecoderLib/VLCReader.cpp | 7 +++++++
source/Lib/EncoderLib/VLCWriter.cpp | 7 +++++++
5 files changed, 57 insertions(+), 6 deletions(-)
diff --git a/source/Lib/CommonLib/DepQuant.cpp b/source/Lib/CommonLib/DepQuant.cpp
index 5bd804176b..e7e06d87da 100644
--- a/source/Lib/CommonLib/DepQuant.cpp
+++ b/source/Lib/CommonLib/DepQuant.cpp
@@ -886,6 +886,7 @@ namespace DQIntern
const int transformShift = ( clipTransformShift ? std::max<int>( 0, nomTransformShift ) : nomTransformShift ) + (needsSqrt2ScaleAdjustment?-1:0);
#if JVET_N0847_SCALING_LISTS
Intermediate_Int shift = IQUANT_SHIFT + 1 - qpPer - transformShift + (enableScalingLists ? LOG2_SCALING_LIST_NEUTRAL_VALUE : 0);
+ Intermediate_Int shiftOrig = shift;
#else
Intermediate_Int shift = IQUANT_SHIFT + 1 - qpPer - transformShift;
#endif
@@ -920,7 +921,10 @@ namespace DQIntern
{
#if JVET_N0847_SCALING_LISTS
if (enableScalingLists)
+ {
invQScale = piDequantCoef[rasterPos];//scalingfactor*levelScale
+ shift = shiftOrig;
+ }
if (shift < 0)
{
invQScale <<= -shift;
@@ -929,8 +933,13 @@ namespace DQIntern
}
#endif
Intermediate_Int qIdx = ( level << 1 ) + ( level > 0 ? -(state>>1) : (state>>1) );
+#if JVET_N0847_SCALING_LISTS
+ int64_t nomTCoeff = ( (int64_t)qIdx * (int64_t)invQScale + add ) >> shift;
+ tCoeff[ rasterPos ] = (TCoeff)Clip3<int64_t>( minTCoeff, maxTCoeff, nomTCoeff );
+#else
Intermediate_Int nomTCoeff = ( qIdx * invQScale + add ) >> shift;
tCoeff[ rasterPos ] = (TCoeff)Clip3<Intermediate_Int>( minTCoeff, maxTCoeff, nomTCoeff );
+#endif
}
state = ( 32040 >> ((state<<2)+((level&1)<<1)) ) & 3; // the 16-bit value "32040" represent the state transition table
}
@@ -1920,7 +1929,7 @@ void DepQuant::quant( TransformUnit &tu, const ComponentID &compID, const CCoeff
CHECK(scalingListType >= SCALING_LIST_NUM, "Invalid scaling list");
const uint32_t log2TrWidth = g_aucLog2[width];
const uint32_t log2TrHeight = g_aucLog2[height];
- const bool enableScalingLists = getUseScalingList(width, height, tu.mtsIdx == MTS_SKIP);
+ const bool enableScalingLists = getUseScalingList(width, height, false);
static_cast<DQIntern::DepQuant*>(p)->quant( tu, pSrc, compID, cQP, Quant::m_dLambda, ctx, uiAbsSum, enableScalingLists, Quant::getQuantCoeff(scalingListType, qpRem, log2TrWidth, log2TrHeight) );
#else
static_cast<DQIntern::DepQuant*>(p)->quant( tu, pSrc, compID, cQP, Quant::m_dLambda, ctx, uiAbsSum );
@@ -1952,7 +1961,7 @@ void DepQuant::dequant( const TransformUnit &tu, CoeffBuf &dstCoeff, const Compo
const uint32_t log2TrWidth = g_aucLog2[width];
const uint32_t log2TrHeight = g_aucLog2[height];
- const bool enableScalingLists = getUseScalingList(width, height, (tu.mtsIdx == MTS_SKIP));
+ const bool enableScalingLists = getUseScalingList(width, height, false);
static_cast<DQIntern::DepQuant*>(p)->dequant( tu, dstCoeff, compID, cQP, enableScalingLists, Quant::getDequantCoeff(scalingListType, qpRem, log2TrWidth, log2TrHeight) );
#else
static_cast<DQIntern::DepQuant*>(p)->dequant( tu, dstCoeff, compID, cQP );
diff --git a/source/Lib/CommonLib/Quant.cpp b/source/Lib/CommonLib/Quant.cpp
index 8950a4adce..72602f2d16 100644
--- a/source/Lib/CommonLib/Quant.cpp
+++ b/source/Lib/CommonLib/Quant.cpp
@@ -452,7 +452,11 @@ void Quant::dequant(const TransformUnit &tu,
const uint32_t uiLog2TrWidth = g_aucLog2[uiWidth];
const uint32_t uiLog2TrHeight = g_aucLog2[uiHeight];
+#if JVET_N0847_SCALING_LISTS
+ int *piDequantCoef = getDequantCoeff(scalingListType, QP_rem, uiLog2TrWidth, uiLog2TrHeight);
+#else
int *piDequantCoef = getDequantCoeff(scalingListType, QP_rem, uiLog2TrWidth - 1, uiLog2TrHeight - 1);
+#endif
if(rightShift > 0)
{
@@ -1024,7 +1028,7 @@ void Quant::quant(TransformUnit &tu, const ComponentID &compID, const CCoeffBuf
const CCoeffBuf &piCoef = pSrc;
CoeffBuf piQCoef = tu.getCoeffs(compID);
- const bool useTransformSkip = tu.mtsIdx==MTS_SKIP;
+ const bool useTransformSkip = tu.mtsIdx==MTS_SKIP && isLuma(compID);
const int maxLog2TrDynamicRange = sps.getMaxLog2TrDynamicRange(toChannelType(compID));
{
@@ -1045,7 +1049,11 @@ void Quant::quant(TransformUnit &tu, const ComponentID &compID, const CCoeffBuf
CHECK(scalingListType >= SCALING_LIST_NUM, "Invalid scaling list");
const uint32_t uiLog2TrWidth = g_aucLog2[uiWidth];
const uint32_t uiLog2TrHeight = g_aucLog2[uiHeight];
+#if JVET_N0847_SCALING_LISTS
+ int *piQuantCoeff = getQuantCoeff(scalingListType, cQP.rem, uiLog2TrWidth, uiLog2TrHeight);
+#else
int *piQuantCoeff = getQuantCoeff(scalingListType, cQP.rem, uiLog2TrWidth-1, uiLog2TrHeight-1);
+#endif
const bool enableScalingLists = getUseScalingList(uiWidth, uiHeight, useTransformSkip);
#endif
@@ -1152,7 +1160,7 @@ bool Quant::xNeedRDOQ(TransformUnit &tu, const ComponentID &compID, const CCoeff
const CCoeffBuf piCoef = pSrc;
- const bool useTransformSkip = tu.mtsIdx==MTS_SKIP;
+ const bool useTransformSkip = tu.mtsIdx==MTS_SKIP && isLuma(compID);
const int maxLog2TrDynamicRange = sps.getMaxLog2TrDynamicRange(toChannelType(compID));
#if HEVC_USE_SCALING_LISTS
@@ -1161,7 +1169,11 @@ bool Quant::xNeedRDOQ(TransformUnit &tu, const ComponentID &compID, const CCoeff
const uint32_t uiLog2TrWidth = g_aucLog2[uiWidth];
const uint32_t uiLog2TrHeight = g_aucLog2[uiHeight];
+#if JVET_N0847_SCALING_LISTS
+ int *piQuantCoeff = getQuantCoeff(scalingListType, cQP.rem, uiLog2TrWidth, uiLog2TrHeight);
+#else
int *piQuantCoeff = getQuantCoeff(scalingListType, cQP.rem, uiLog2TrWidth-1, uiLog2TrHeight-1);
+#endif
const bool enableScalingLists = getUseScalingList(uiWidth, uiHeight, (useTransformSkip != 0));
#endif
@@ -1262,7 +1274,11 @@ void Quant::transformSkipQuantOneSample(TransformUnit &tu, const ComponentID &co
const uint32_t uiLog2TrWidth = g_aucLog2[uiWidth];
const uint32_t uiLog2TrHeight = g_aucLog2[uiHeight];
+#if JVET_N0847_SCALING_LISTS
+ const int *const piQuantCoeff = getQuantCoeff(scalingListType, cQP.rem, uiLog2TrWidth, uiLog2TrHeight);
+#else
const int *const piQuantCoeff = getQuantCoeff(scalingListType, cQP.rem, uiLog2TrWidth-1, uiLog2TrHeight-1);
+#endif
#endif
/* for 422 chroma blocks, the effective scaling applied during transformation is not a power of 2, hence it cannot be
@@ -1348,7 +1364,11 @@ void Quant::invTrSkipDeQuantOneSample(TransformUnit &tu, const ComponentID &comp
const uint32_t uiLog2TrWidth = g_aucLog2[uiWidth];
const uint32_t uiLog2TrHeight = g_aucLog2[uiHeight];
+#if JVET_N0847_SCALING_LISTS
+ int *piDequantCoef = getDequantCoeff(scalingListType,QP_rem,uiLog2TrWidth, uiLog2TrHeight);
+#else
int *piDequantCoef = getDequantCoeff(scalingListType,QP_rem,uiLog2TrWidth-1, uiLog2TrHeight-1);
+#endif
if (rightShift > 0)
{
diff --git a/source/Lib/CommonLib/QuantRDOQ.cpp b/source/Lib/CommonLib/QuantRDOQ.cpp
index c00ce5c08c..7f7efd7cc3 100644
--- a/source/Lib/CommonLib/QuantRDOQ.cpp
+++ b/source/Lib/CommonLib/QuantRDOQ.cpp
@@ -576,7 +576,7 @@ void QuantRDOQ::quant(TransformUnit &tu, const ComponentID &compID, const CCoeff
const CCoeffBuf &piCoef = pSrc;
CoeffBuf piQCoef = tu.getCoeffs(compID);
- const bool useTransformSkip = tu.mtsIdx==MTS_SKIP;
+ const bool useTransformSkip = tu.mtsIdx==MTS_SKIP && isLuma(compID);
bool useRDOQ = useTransformSkip ? m_useRDOQTS : m_useRDOQ;
@@ -704,15 +704,23 @@ void QuantRDOQ::xRateDistOptQuant(TransformUnit &tu, const ComponentID &compID,
const bool needSqrtAdjustment= TU::needsBlockSizeTrafoScale( tu, compID );
#if HEVC_USE_SCALING_LISTS
#if JVET_N0847_SCALING_LISTS
- const double *const pdErrScale = xGetErrScaleCoeffSL(scalingListType, (uiLog2BlockWidth - 1), (uiLog2BlockHeight - 1), cQP.rem);
+ const double *const pdErrScale = xGetErrScaleCoeffSL(scalingListType, uiLog2BlockWidth, uiLog2BlockHeight, cQP.rem);
#else
const double *const pdErrScale = xGetErrScaleCoeff(scalingListType, (uiLog2BlockWidth-1), (uiLog2BlockHeight-1), cQP.rem);
#endif
+#if JVET_N0847_SCALING_LISTS
+ const int *const piQCoef = getQuantCoeff(scalingListType, cQP.rem, (uiLog2BlockWidth), (uiLog2BlockHeight));
+#else
const int *const piQCoef = getQuantCoeff(scalingListType, cQP.rem, (uiLog2BlockWidth-1), (uiLog2BlockHeight-1));
+#endif
const bool isTransformSkip = tu.mtsIdx==MTS_SKIP && isLuma(compID);
const bool enableScalingLists = getUseScalingList(uiWidth, uiHeight, isTransformSkip);
const int defaultQuantisationCoefficient = g_quantScales[ needSqrtAdjustment ?1:0][cQP.rem];
+#if JVET_N0847_SCALING_LISTS
+ const double defaultErrorScale = xGetErrScaleCoeffNoScalingList(scalingListType, uiLog2BlockWidth, uiLog2BlockHeight, cQP.rem);
+#else
const double defaultErrorScale = xGetErrScaleCoeffNoScalingList(scalingListType, (uiLog2BlockWidth-1), (uiLog2BlockHeight-1), cQP.rem);
+#endif
const int iQBits = QUANT_SHIFT + cQP.per + iTransformShift + (needSqrtAdjustment?-1:0); // Right shift of non-RDOQ quantizer; level = (coeff*uiQ + offset)>>q_bits
#else
const int quantisationCoefficient = g_quantScales[needSqrtAdjustment?1:0][cQP.rem];
diff --git a/source/Lib/DecoderLib/VLCReader.cpp b/source/Lib/DecoderLib/VLCReader.cpp
index f43efc759b..f441680e1f 100644
--- a/source/Lib/DecoderLib/VLCReader.cpp
+++ b/source/Lib/DecoderLib/VLCReader.cpp
@@ -2891,6 +2891,13 @@ void HLSyntaxReader::parseScalingList(ScalingList* scalingList)
#endif
}
+#if JVET_N0847_SCALING_LISTS
+ // adjust the decoded code for 2x2 to cope with the missing 2x2 luma entries
+ if (sizeId == SCALING_LIST_2x2 && ((listId==4 && code>0) || (listId==5 && code>1)))
+ {
+ code++;
+ }
+#endif
scalingList->setRefMatrixId (sizeId,listId,(uint32_t)((int)(listId)-(code)));
if( sizeId > SCALING_LIST_8x8 )
{
diff --git a/source/Lib/EncoderLib/VLCWriter.cpp b/source/Lib/EncoderLib/VLCWriter.cpp
index 90bce51511..10ce335a8b 100644
--- a/source/Lib/EncoderLib/VLCWriter.cpp
+++ b/source/Lib/EncoderLib/VLCWriter.cpp
@@ -2085,6 +2085,13 @@ void HLSWriter::codeScalingList( const ScalingList &scalingList )
WRITE_UVLC( ((int)listId - (int)scalingList.getRefMatrixId (sizeId,listId)) / (SCALING_LIST_NUM/NUMBER_OF_PREDICTION_MODES), "scaling_list_pred_matrix_id_delta");
#endif
}
+#if JVET_N0847_SCALING_LISTS
+ // adjust the code for 2x2 to cope with the missing 2x2 luma entries
+ else if (sizeId == SCALING_LIST_2x2 && listId > 3 && scalingList.getRefMatrixId(sizeId, listId) < 3)
+ {
+ WRITE_UVLC( (int)listId - (int)scalingList.getRefMatrixId (sizeId,listId) - 1, "scaling_list_pred_matrix_id_delta");
+ }
+#endif
else
{
WRITE_UVLC( (int)listId - (int)scalingList.getRefMatrixId (sizeId,listId), "scaling_list_pred_matrix_id_delta");
--
GitLab