diff --git a/source/Lib/CommonLib/DepQuant.cpp b/source/Lib/CommonLib/DepQuant.cpp
index a682bad02c55a0b79a01b4818a08e1211a8b7786..bbc6d72d74c3256df3ef575b3c07eed56a028156 100644
--- a/source/Lib/CommonLib/DepQuant.cpp
+++ b/source/Lib/CommonLib/DepQuant.cpp
@@ -753,12 +753,10 @@ namespace DQIntern
if (shift < 0)
{
invQScale <<= -shift;
- shift = 0;
- //add = (1 << shift) >> 1;
}
Intermediate_Int qIdx = ( level << 1 ) + ( level > 0 ? -(state>>1) : (state>>1) );
- Intermediate_Int nomTCoeff = ( qIdx * invQScale + add ) >> shift;
- tCoeff[ rasterPos ] = (TCoeff)Clip3<Intermediate_Int>( minTCoeff, maxTCoeff, nomTCoeff );
+ int64_t nomTCoeff = ((int64_t)qIdx * (int64_t)invQScale + add) >> ((shift < 0) ? 0 : shift);
+ tCoeff[rasterPos] = (TCoeff)Clip3<int64_t>(minTCoeff, maxTCoeff, nomTCoeff);
}
state = ( 32040 >> ((state<<2)+((level&1)<<1)) ) & 3; // the 16-bit value "32040" represent the state transition table
}
@@ -1615,7 +1613,7 @@ void DepQuant::quant( TransformUnit &tu, const ComponentID &compID, const CCoeff
CHECK(scalingListType >= SCALING_LIST_NUM, "Invalid scaling list");
const uint32_t log2TrWidth = g_aucLog2[width];
const uint32_t log2TrHeight = g_aucLog2[height];
- const bool enableScalingLists = getUseScalingList(width, height, tu.mtsIdx == MTS_SKIP);
+ const bool enableScalingLists = getUseScalingList(width, height, (tu.mtsIdx == MTS_SKIP && isLuma(compID)));
static_cast<DQIntern::DepQuant*>(p)->quant( tu, pSrc, compID, cQP, Quant::m_dLambda, ctx, uiAbsSum, enableScalingLists, Quant::getQuantCoeff(scalingListType, qpRem, log2TrWidth, log2TrHeight) );
}
else
@@ -1639,7 +1637,7 @@ void DepQuant::dequant( const TransformUnit &tu, CoeffBuf &dstCoeff, const Compo
const uint32_t log2TrWidth = g_aucLog2[width];
const uint32_t log2TrHeight = g_aucLog2[height];
- const bool enableScalingLists = getUseScalingList(width, height, (tu.mtsIdx == MTS_SKIP));
+ const bool enableScalingLists = getUseScalingList(width, height, (tu.mtsIdx == MTS_SKIP && isLuma(compID)));
static_cast<DQIntern::DepQuant*>(p)->dequant( tu, dstCoeff, compID, cQP, enableScalingLists, Quant::getDequantCoeff(scalingListType, qpRem, log2TrWidth, log2TrHeight) );
}
else
diff --git a/source/Lib/CommonLib/Quant.cpp b/source/Lib/CommonLib/Quant.cpp
index faffc57c5d3bd1ccb2c3cecefec79ac0e36af6bf..43f78293fc525b92ed90b3054326cf4092fe1388 100644
--- a/source/Lib/CommonLib/Quant.cpp
+++ b/source/Lib/CommonLib/Quant.cpp
@@ -399,7 +399,7 @@ void Quant::dequant(const TransformUnit &tu,
const uint32_t uiLog2TrWidth = g_aucLog2[uiWidth];
const uint32_t uiLog2TrHeight = g_aucLog2[uiHeight];
- int *piDequantCoef = getDequantCoeff(scalingListType, QP_rem, uiLog2TrWidth - 1, uiLog2TrHeight - 1);
+ int *piDequantCoef = getDequantCoeff(scalingListType, QP_rem, uiLog2TrWidth, uiLog2TrHeight);
if(rightShift > 0)
{
@@ -913,7 +913,7 @@ void Quant::quant(TransformUnit &tu, const ComponentID &compID, const CCoeffBuf
const CCoeffBuf &piCoef = pSrc;
CoeffBuf piQCoef = tu.getCoeffs(compID);
- const bool useTransformSkip = tu.mtsIdx==MTS_SKIP;
+ const bool useTransformSkip = tu.mtsIdx==MTS_SKIP && isLuma(compID);
const int maxLog2TrDynamicRange = sps.getMaxLog2TrDynamicRange(toChannelType(compID));
{
@@ -927,7 +927,7 @@ void Quant::quant(TransformUnit &tu, const ComponentID &compID, const CCoeffBuf
CHECK(scalingListType >= SCALING_LIST_NUM, "Invalid scaling list");
const uint32_t uiLog2TrWidth = g_aucLog2[uiWidth];
const uint32_t uiLog2TrHeight = g_aucLog2[uiHeight];
- int *piQuantCoeff = getQuantCoeff(scalingListType, cQP.rem, uiLog2TrWidth-1, uiLog2TrHeight-1);
+ int *piQuantCoeff = getQuantCoeff(scalingListType, cQP.rem, uiLog2TrWidth, uiLog2TrHeight);
const bool enableScalingLists = getUseScalingList(uiWidth, uiHeight, useTransformSkip);
@@ -990,7 +990,7 @@ bool Quant::xNeedRDOQ(TransformUnit &tu, const ComponentID &compID, const CCoeff
const CCoeffBuf piCoef = pSrc;
- const bool useTransformSkip = tu.mtsIdx==MTS_SKIP;
+ const bool useTransformSkip = tu.mtsIdx == MTS_SKIP && isLuma(compID);
const int maxLog2TrDynamicRange = sps.getMaxLog2TrDynamicRange(toChannelType(compID));
int scalingListType = getScalingListType(tu.cu->predMode, compID);
@@ -998,7 +998,7 @@ bool Quant::xNeedRDOQ(TransformUnit &tu, const ComponentID &compID, const CCoeff
const uint32_t uiLog2TrWidth = g_aucLog2[uiWidth];
const uint32_t uiLog2TrHeight = g_aucLog2[uiHeight];
- int *piQuantCoeff = getQuantCoeff(scalingListType, cQP.rem, uiLog2TrWidth-1, uiLog2TrHeight-1);
+ int *piQuantCoeff = getQuantCoeff(scalingListType, cQP.rem, uiLog2TrWidth, uiLog2TrHeight);
const bool enableScalingLists = getUseScalingList(uiWidth, uiHeight, (useTransformSkip != 0));
@@ -1056,7 +1056,7 @@ void Quant::transformSkipQuantOneSample(TransformUnit &tu, const ComponentID &co
const uint32_t uiLog2TrWidth = g_aucLog2[uiWidth];
const uint32_t uiLog2TrHeight = g_aucLog2[uiHeight];
- const int *const piQuantCoeff = getQuantCoeff(scalingListType, cQP.rem, uiLog2TrWidth-1, uiLog2TrHeight-1);
+ const int *const piQuantCoeff = getQuantCoeff(scalingListType, cQP.rem, uiLog2TrWidth, uiLog2TrHeight);
/* for 422 chroma blocks, the effective scaling applied during transformation is not a power of 2, hence it cannot be
* implemented as a bit-shift (the quantised result will be sqrt(2) * larger than required). Alternatively, adjust the
@@ -1130,7 +1130,7 @@ void Quant::invTrSkipDeQuantOneSample(TransformUnit &tu, const ComponentID &comp
const uint32_t uiLog2TrWidth = g_aucLog2[uiWidth];
const uint32_t uiLog2TrHeight = g_aucLog2[uiHeight];
- int *piDequantCoef = getDequantCoeff(scalingListType,QP_rem,uiLog2TrWidth-1, uiLog2TrHeight-1);
+ int *piDequantCoef = getDequantCoeff(scalingListType, QP_rem, uiLog2TrWidth, uiLog2TrHeight);
if (rightShift > 0)
{
diff --git a/source/Lib/CommonLib/QuantRDOQ.cpp b/source/Lib/CommonLib/QuantRDOQ.cpp
index 82039dd613dd2b4b49afd3eba8f60220f43919ea..79929415e03a9a69c06166ea8e5ffb473abfab1e 100644
--- a/source/Lib/CommonLib/QuantRDOQ.cpp
+++ b/source/Lib/CommonLib/QuantRDOQ.cpp
@@ -511,7 +511,7 @@ void QuantRDOQ::quant(TransformUnit &tu, const ComponentID &compID, const CCoeff
const CCoeffBuf &piCoef = pSrc;
CoeffBuf piQCoef = tu.getCoeffs(compID);
- const bool useTransformSkip = tu.mtsIdx==MTS_SKIP;
+ const bool useTransformSkip = tu.mtsIdx==MTS_SKIP && isLuma(compID);
bool useRDOQ = useTransformSkip ? m_useRDOQTS : m_useRDOQ;
@@ -620,8 +620,8 @@ void QuantRDOQ::xRateDistOptQuant(TransformUnit &tu, const ComponentID &compID,
const bool needSqrtAdjustment= TU::needsBlockSizeTrafoScale( tu, compID );
- const double *const pdErrScale = xGetErrScaleCoeffSL(scalingListType, (uiLog2BlockWidth - 1), (uiLog2BlockHeight - 1), cQP.rem);
- const int *const piQCoef = getQuantCoeff(scalingListType, cQP.rem, (uiLog2BlockWidth-1), (uiLog2BlockHeight-1));
+ const double *const pdErrScale = xGetErrScaleCoeffSL(scalingListType, uiLog2BlockWidth, uiLog2BlockHeight, cQP.rem);
+ const int *const piQCoef = getQuantCoeff(scalingListType, cQP.rem, uiLog2BlockWidth, uiLog2BlockHeight);
const bool isTransformSkip = tu.mtsIdx==MTS_SKIP && isLuma(compID);
const bool enableScalingLists = getUseScalingList(uiWidth, uiHeight, isTransformSkip);
const int defaultQuantisationCoefficient = g_quantScales[ needSqrtAdjustment ?1:0][cQP.rem];
diff --git a/source/Lib/CommonLib/Slice.cpp b/source/Lib/CommonLib/Slice.cpp
index ede292b45cfb96190248df40ce8c26f1deac60ce..33080f337ccb9116b17b0abf4966cf5f0347820d 100644
--- a/source/Lib/CommonLib/Slice.cpp
+++ b/source/Lib/CommonLib/Slice.cpp
@@ -1731,7 +1731,7 @@ void ScalingList::checkPredMode(uint32_t sizeId, uint32_t listId)
{
for (int predListIdx = (int)listId; predListIdx >= 0; predListIdx--)
{
- if ((sizeId == SCALING_LIST_64x64 && (listId % 3) != 0) || (sizeId == SCALING_LIST_2x2 && (listId % 3) == 0))
+ if ((sizeId == SCALING_LIST_64x64 && ((listId % 3) != 0 || (predListIdx % 3) != 0)) || (sizeId == SCALING_LIST_2x2 && ((listId % 3) == 0 || (predListIdx % 3) == 0)))
continue;
if( !::memcmp(getScalingListAddress(sizeId,listId),((listId == predListIdx) ?
getScalingListDefaultAddress(sizeId, predListIdx): getScalingListAddress(sizeId, predListIdx)),sizeof(int)*std::min(MAX_MATRIX_COEF_NUM,(int)g_scalingListSize[sizeId])) // check value of matrix