Merge pull request #90 from datajoint/stage

add flag for reading as 32-bit

.github/workflows/integration.yml

@@ -19,8 +19,8 @@ jobs:
         include:
           - matlab_version: "R2018b"
             mysql_version: "5.7"
-          - matlab_version: "R2016b"
-            mysql_version: "5.7"
+          # - matlab_version: "R2016b"
+          #   mysql_version: "5.7"
     steps:
       - uses: actions/checkout@v2
       - name: Run primary tests

LNX-docker-compose.yml

@@ -11,7 +11,7 @@ services:
       - MYSQL_ROOT_PASSWORD=simple
   fakeservices.datajoint.io:
     <<: *net
-    image: datajoint/nginx:v0.0.16
+    image: datajoint/nginx:v0.1.1
     environment:
     - ADD_db_TYPE=DATABASE
     - ADD_db_ENDPOINT=db:3306

local-docker-compose.yml

@@ -16,7 +16,7 @@ services:
     #   - ./mysql/data:/var/lib/mysql
   fakeservices.datajoint.io:
     <<: *net
-    image: datajoint/nginx:v0.0.16
+    image: datajoint/nginx:v0.1.1
     environment:
     - ADD_db_TYPE=DATABASE
     - ADD_db_ENDPOINT=db:3306

src/mym.cpp

@@ -52,8 +52,13 @@ typedef size_t mwIndex;
 
 //The crazy do{}while(0) constructions circumvents unexpected results when using the macro followed by a semicolon in
 //an if/else construction
-#define READ_UINT64(dst,src) do{ safe_read_64uint( (dst), (_uint64*)(src), 1 );   (src) += sizeof(_uint64);  } while(0)
-#define READ_UINT64S(dst,src,n) do{ safe_read_64uint( (dst), (_uint64*)(src), n );   (src) += (n) * sizeof(_uint64);  } while(0)
+#define READ_UINT(dst,src) \
+    if (use32bitdims()) do{ safe_read_32uint( (dst), (_uint32*)(src), 1 );   (src) += sizeof(_uint32);  } while(0); \
+    else do{ safe_read_64uint( (dst), (_uint64*)(src), 1 );   (src) += sizeof(_uint64);  } while(0)
+#define READ_UINTS(dst,src,n) \
+    if (use32bitdims()) do{ safe_read_32uint( (dst), (_uint32*)(src), n );   (src) += (n) * sizeof(_uint32);  } while(0); \
+    else do{ safe_read_64uint( (dst), (_uint64*)(src), n );   (src) += (n) * sizeof(_uint64);  } while(0)
+
 // Macro to write fixed size 64 bit uints
 #define WRITE_UINT64(p,val) do{  *((_uint64*)(p)) = (_uint64)(val);    (p) += sizeof(_uint64);  }while(0)
 #define WRITE_UINT64S(p,val,n) do{     _uint64* pTemp = (_uint64*) (p); \
@@ -353,6 +358,14 @@ static void updateplugindir() {
     mxDestroyArray(mym_fileparts[1]);
     mxDestroyArray(mym_fileparts[2]);
 }
+/**********************************************************************
+ *use32bitdims():   Check if dimensions should be read as 32-bit
+ *  Get MYM_USE_32BIT_DIMS environment variable
+ *  Return true/false based on MYM_USE_32BIT_DIMS flag
+ **********************************************************************/
+static bool use32bitdims() {
+    return getenv("MYM_USE_32BIT_DIMS") && strcasecmp(getenv("MYM_USE_32BIT_DIMS"), "true") == 0;
+}
 /**********************************************************************
  * mysql():  Execute the actual action
  * Which action we perform is based on the first input argument,
@@ -1727,11 +1740,11 @@ mxArray* deserializeArray(const char *rpSerial, const size_t rlength)
 
     // number of dimensions
     mwSize ndims;
-    READ_UINT64(&ndims, p_serial);
+    READ_UINT(&ndims, p_serial);
 
     // length of each dimension
     mwSize *pdims = (mwSize*) mxCalloc(ndims, sizeof(mwSize));
-    READ_UINT64S(pdims, p_serial, ndims);
+    READ_UINTS(pdims, p_serial, ndims);
 
     // classID
     mxClassID class_id;
@@ -1770,9 +1783,9 @@ mxArray* deserializeSparse(const char *rpSerial, const size_t rlength)
 
     // number of rows and columns
     mwSize rows, columns, nzmax;
-    READ_UINT64(&rows, p_serial);
-    READ_UINT64(&columns, p_serial);
-    READ_UINT64(&nzmax, p_serial);
+    READ_UINT(&rows, p_serial);
+    READ_UINT(&columns, p_serial);
+    READ_UINT(&nzmax, p_serial);
 
     // classID
     mxClassID class_id;
@@ -1796,11 +1809,11 @@ mxArray* deserializeSparse(const char *rpSerial, const size_t rlength)
 
     // JC data
     mwSize* pJcData = mxGetJc(p_array);
-    READ_UINT64S(pJcData, p_serial, columns+1);
+    READ_UINTS(pJcData, p_serial, columns+1);
 
     // size of IR array
     mwSize* pIrData = mxGetIr(p_array);
-    READ_UINT64S(pIrData, p_serial, nzmax);
+    READ_UINTS(pIrData, p_serial, nzmax);
 
     // real part data
     void* pdata = mxGetData(p_array);
@@ -1823,11 +1836,11 @@ mxArray* deserializeStruct(const char* rpSerial, const size_t rlength){
     const char*p_serial = rpSerial;
     // number of dimensions
     mwSize ndims;
-    READ_UINT64(&ndims, p_serial);
+    READ_UINT(&ndims, p_serial);
 
     // length of each dimension
     mwSize *pdims = (mwSize*) mxCalloc(ndims, sizeof(mwSize));
-    READ_UINT64S(pdims, p_serial, ndims);
+    READ_UINTS(pdims, p_serial, ndims);
 
     mwSize nelts = 1;
     for (int i = 0; i<ndims; i++)
@@ -1849,7 +1862,7 @@ mxArray* deserializeStruct(const char* rpSerial, const size_t rlength){
         for (int j = 0; j<nfields; j++) {
             mwSize nbytes1;
             size_t nbytes;
-            READ_UINT64(&nbytes1, p_serial);
+            READ_UINT(&nbytes1, p_serial);
             nbytes = nbytes1;
             mxArray* pf;
             if (*p_serial==ID_ARRAY)
@@ -1874,11 +1887,11 @@ mxArray* deserializeCell(const char* rpSerial, const size_t rlength){
     const char* p_serial = rpSerial;
     // number of dimensions
     mwSize ndims;
-    READ_UINT64(&ndims, p_serial);
+    READ_UINT(&ndims, p_serial);
 
     // length of each dimension
     mwSize *pdims = (mwSize*) mxCalloc(ndims, sizeof(mwSize));
-    READ_UINT64S(pdims, p_serial, ndims);
+    READ_UINTS(pdims, p_serial, ndims);
 
     mwSize nelts = 1;
     for (int i = 0; i<ndims; i++)
@@ -1889,7 +1902,7 @@ mxArray* deserializeCell(const char* rpSerial, const size_t rlength){
     for (int i = 0; i<nelts; i++){
         size_t nbytes;
         mwSize nbytes1;
-        READ_UINT64(&nbytes1, p_serial);
+        READ_UINT(&nbytes1, p_serial);
         nbytes=nbytes1;
         mxArray* pf;
         if (*p_serial==ID_ARRAY)
@@ -1939,6 +1952,12 @@ void safe_read_64uint(mwSize* dst, _uint64* src, size_t n) {
     }
 }
 
+void safe_read_32uint(mwSize* dst, _uint32* src, size_t n) {
+    for (size_t i=0; i < n; ++i) {
+            dst[i] = mwSize(src[i]);
+    }
+}
+
 static void getSerialFct(const char* rpt, const mxArray* rparg, pfserial& rpf, bool& rpec) {
     const mwSize n_dims = mxGetNumberOfDimensions(rparg);
     const mwSize* p_dim = mxGetDimensions(rparg);
@@ -2005,7 +2024,7 @@ mxArray* deserialize(const char* rpSerial, const size_t rlength) {
         // read the length in bytes
         mwSize len;
         size_t lenLong;
-        READ_UINT64(&len, p_serial);
+        READ_UINT(&len, p_serial);
         char* p_cmp = (char*)mxCalloc(len, sizeof(char));
         lenLong=len;
         try {

src/mym.h

@@ -34,7 +34,7 @@
 // mym version information
 #define MYM_VERSION_MAJOR 2
 #define MYM_VERSION_MINOR 8
-#define MYM_VERSION_BUGFIX 2
+#define MYM_VERSION_BUGFIX 5
 
 
 // some local defintion
@@ -57,9 +57,13 @@
 	#include <sys/types.h>
 	#define _uint64 uint64_t
 	#define _int64  int64_t
+	#define _uint32 uint32_t
+	#define _int32  int32_t
 #elif _MSC_VER
 	#define _uint64 unsigned __int64
 	#define _int64 __int64
+	#define _uint32 unsigned __int32
+	#define _int32 __int32
 	#define _WINDOWS 1
 #else
 	#error "I don't know how to declare a 64 bit uint for this compiler. Please fix!"
@@ -106,6 +110,7 @@ mxArray* deserializeCell(const char* rpSerial, const size_t rlength);
 int file_length(FILE *f); // get the size of a file in byte
 unsigned long min_mysql_escape(char* rpout, const char* rpin, const unsigned long nin);
 void safe_read_64uint(mwSize* dst, _uint64* src, size_t n);
+void safe_read_32uint(mwSize* dst, _uint32* src, size_t n);
 bool isSubstringFountAtTheBeginningCaseInsenstive(const char* sourceString, const char* subString);
 void removeWhiteSpaceAtTheBeginning(char* string);
 //void safe_read_64uint(mwSize* dst, unsigned __int64* src, size_t n);

tests/TestExternal.m

@@ -90,6 +90,35 @@ function TestExternal_testdj0Error(testCase)
                 testCase.verifyEqual(ME.identifier, 'mYm:CrossPlatform:Compatibility');
             end
         end
+        function TestExternal_test32BitRead(testCase)
+            st = dbstack;
+            disp(['---------------' st(1).name '---------------']);
+            % https://github.com/datajoint/mym/issues/86
+            value = ['6D596D005302000000010000000100000004000000686974730073696465730074' ...
+                '61736B73007374616765004D00000041020000000100000007000000060000000000000' ...
+                '0000000000000F8FF000000000000F03F000000000000F03F0000000000000000000000' ...
+                '000000F03F0000000000000000000000000000F8FF23000000410200000001000000070' ...
+                '0000004000000000000006C006C006C006C00720072006C002300000041020000000100' ...
+                '00000700000004000000000000006400640064006400640064006400250000004102000' ...
+                '0000100000008000000040000000000000053007400610067006500200031003000'];
+            hexstring = value';
+            reshapedString = reshape(hexstring,2,length(value)/2);
+            hexMtx = reshapedString.';
+            decMtx = hex2dec(hexMtx);
+            packed = uint8(decMtx);
+
+            data = struct;
+            data.stage = 'Stage 10';
+            data.tasks = 'ddddddd';
+            data.sides = 'llllrrl';
+            data.hits = [NaN,1,1,0,1,0,NaN];
+
+            setenv('MYM_USE_32BIT_DIMS', 'true');
+            unpacked = mym('deserialize', packed);
+            setenv('MYM_USE_32BIT_DIMS', 'false');
+
+            testCase.verifyEqual(unpacked, data);
+        end
     end
     methods (Static)
         function TestExternal_verify(testCase, array_tests)