Skip to content

Commit cfd5500

Browse files
committed
fix z component storing too many bits in quantized sequence in vec2 data type for dim 3
1 parent 965e028 commit cfd5500

File tree

2 files changed

+8
-6
lines changed

2 files changed

+8
-6
lines changed

include/nbl/builtin/hlsl/sampling/quantized_sequence.hlsl

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -164,22 +164,24 @@ struct QuantizedSequence<T, Dim NBL_PARTIAL_REQ_BOT(SEQUENCE_SPECIALIZATION_CONC
164164

165165
scalar_type get(const uint16_t idx)
166166
{
167-
// assert(idx >= 0 && idx < 3);
167+
assert(idx >= 0 && idx < 3);
168168
if (idx < 2)
169169
{
170170
return data[idx] & Mask;
171171
}
172172
else
173173
{
174-
scalar_type z = data[0] >> BitsPerComponent;
175-
z |= (data[1] >> BitsPerComponent) << DiscardBits;
174+
const scalar_type zbits = scalar_type(DiscardBits);
175+
const scalar_type zmask = (scalar_type(1u) << zbits) - scalar_type(1u);
176+
scalar_type z = (data[0] >> BitsPerComponent) & zmask;
177+
z |= ((data[1] >> BitsPerComponent) & zmask) << DiscardBits;
176178
return z;
177179
}
178180
}
179181

180182
void set(const uint16_t idx, const scalar_type value)
181183
{
182-
// assert(idx >= 0 && idx < 3);
184+
assert(idx >= 0 && idx < 3);
183185
if (idx < 2)
184186
{
185187
const scalar_type trunc_val = value >> DiscardBits;
@@ -190,7 +192,7 @@ struct QuantizedSequence<T, Dim NBL_PARTIAL_REQ_BOT(SEQUENCE_SPECIALIZATION_CONC
190192
{
191193
const scalar_type zbits = scalar_type(DiscardBits);
192194
const scalar_type zmask = (scalar_type(1u) << zbits) - scalar_type(1u);
193-
const scalar_type trunc_val = value >> (DiscardBits-1u);
195+
const scalar_type trunc_val = value >> DiscardBits;
194196
data[0] &= Mask;
195197
data[1] &= Mask;
196198
data[0] |= (trunc_val & zmask) << BitsPerComponent;

0 commit comments

Comments
 (0)