From 4726accfc301eec350fecfc929ea731ba2aa5c0f Mon Sep 17 00:00:00 2001 From: spectralcode Date: Sat, 17 Feb 2024 00:35:39 +0100 Subject: [PATCH] scaling factor typo in 24 bit output conversion fixed --- octproz_project/octproz/src/cuda_code.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octproz_project/octproz/src/cuda_code.cu b/octproz_project/octproz/src/cuda_code.cu index debd886..bdc400b 100644 --- a/octproz_project/octproz/src/cuda_code.cu +++ b/octproz_project/octproz/src/cuda_code.cu @@ -916,7 +916,7 @@ __global__ void floatToOutput(void *output, const float *input, const int output out[index] = (unsigned short)(input[index] * (65535.0)); //16 bit }else if(outputBitdepth > 16 && outputBitdepth <= 24){ unsigned int* out = (unsigned int*)output; - out[index] = (unsigned int)(input[index] * (167772165.0f)); //24 bit + out[index] = (unsigned int)(input[index] * (16777215.0f)); //24 bit }else{ unsigned int* out = (unsigned int*)output; out[index] = (unsigned int)(input[index] * (4294967295.0f)); //32 bit