scaling factor typo in 24 bit output conversion fixed

spectralcode · Feb 16, 2024 · 4726acc · 4726acc
1 parent eee5a2d
commit 4726acc
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/octproz_project/octproz/src/cuda_code.cu b/octproz_project/octproz/src/cuda_code.cu
@@ -916,7 +916,7 @@ __global__ void floatToOutput(void *output, const float *input, const int output
 		out[index] = (unsigned short)(input[index] * (65535.0)); //16 bit
 	}else if(outputBitdepth > 16 && outputBitdepth <= 24){
 		unsigned int* out = (unsigned int*)output;
-		out[index] = (unsigned int)(input[index] * (167772165.0f)); //24 bit
+		out[index] = (unsigned int)(input[index] * (16777215.0f)); //24 bit
 	}else{
 		unsigned int* out = (unsigned int*)output;
 		out[index] = (unsigned int)(input[index] * (4294967295.0f)); //32 bit