CXX-479 Backport server r2.6.7..r2.6.9 changes

Server SHAs picked into this commit:

2d45919
1b64aa8
78d52ff
1dd33f4
224067d
8f1c734
91ab64a
a5408ed
738e366
5b0ff61
3abc04d
a34e139
77fb852
b6a66a0
9af0d09
a5a4852
e9e813d
e3a7692
3133d04
2dee1ad
df313bc

SConstruct

@@ -96,7 +96,7 @@ def add_option( name, help, nargs, contributesToVariantDir,
     if type == 'choice' and not metavar:
         metavar = '[' + '|'.join(choices) + ']'
 
-    AddOption( "--" + name , 
+    AddOption( "--" + name ,
                dest=dest,
                type=type,
                nargs=nargs,
@@ -133,7 +133,7 @@ def use_system_version_of_library(name):
 
 def get_variant_dir():
     if has_option('variant-dir'):
-        return "#build/" + get_option('variant-dir') 
+        return "#build/" + get_option('variant-dir')
 
     substitute = lambda x: re.sub( "[:,\\\\/]" , "_" , x )
 
@@ -346,7 +346,7 @@ boostLibs = [ "thread" , "filesystem" , "program_options", "system" ]
 onlyServer = len( COMMAND_LINE_TARGETS ) == 0 or ( len( COMMAND_LINE_TARGETS ) == 1 and str( COMMAND_LINE_TARGETS[0] ) in [ "mongod" , "mongos" , "test" ] )
 
 linux64  = False
-force32 = has_option( "force32" ) 
+force32 = has_option( "force32" )
 force64 = has_option( "force64" )
 if not force64 and not force32 and os.getcwd().endswith( "mongo-64" ):
     force64 = True
@@ -436,9 +436,9 @@ else:
 
 static = has_option( "static" )
 
-noshell = has_option( "noshell" ) 
+noshell = has_option( "noshell" )
 
-usev8 = has_option( "usev8" ) 
+usev8 = has_option( "usev8" )
 
 asio = has_option( "asio" )
 
@@ -582,7 +582,7 @@ if has_option( "libpath" ):
 if has_option( "cpppath" ):
     env["CPPPATH"] = [get_option( "cpppath" )]
 
-env.Prepend( CPPDEFINES=[ "_SCONS" , 
+env.Prepend( CPPDEFINES=[ "_SCONS" ,
                           "MONGO_EXPOSE_MACROS" ,
                           "SUPPORT_UTF8" ],  # for pcre
 
@@ -688,8 +688,6 @@ elif linux:
 
     if static:
         env.Append( LINKFLAGS=" -static " )
-    if has_option( "static-libstdc++" ):
-        env.Append( LINKFLAGS=["-static-libstdc++", "-static-libgcc"] )
 
 elif solaris:
      env.Append( CPPDEFINES=[ "__sunos__" ] )
@@ -739,7 +737,7 @@ elif windows:
     #    The this pointer is valid only within nonstatic member functions. It cannot be used in the initializer list for a base class.
     # c4800
     # 'type' : forcing value to bool 'true' or 'false' (performance warning)
-    #    This warning is generated when a value that is not bool is assigned or coerced into type bool. 
+    #    This warning is generated when a value that is not bool is assigned or coerced into type bool.
     # c4267
     # 'var' : conversion from 'size_t' to 'type', possible loss of data
     # When compiling with /Wp64, or when compiling on a 64-bit operating system, type is 32 bits but size_t is 64 bits when compiling for 64-bit targets. To fix this warning, use size_t instead of a type.
@@ -761,12 +759,12 @@ elif windows:
 
     env.Append( CPPDEFINES=["_CONSOLE","_CRT_SECURE_NO_WARNINGS"] )
 
-    # this would be for pre-compiled headers, could play with it later  
+    # this would be for pre-compiled headers, could play with it later
     #env.Append( CCFLAGS=['/Yu"pch.h"'] )
 
     # docs say don't use /FD from command line (minimal rebuild)
     # /Gy function level linking (implicit when using /Z7)
-    # /Z7 debug info goes into each individual .obj file -- no .pdb created 
+    # /Z7 debug info goes into each individual .obj file -- no .pdb created
     env.Append( CCFLAGS= ["/Z7", "/errorReport:none"] )
 
     # /DEBUG will tell the linker to create a .pdb file
@@ -822,6 +820,9 @@ elif windows:
 
 if nix:
 
+    if has_option( "static-libstdc++" ):
+        env.Append( LINKFLAGS=["-static-libstdc++", "-static-libgcc"] )
+
     if has_option( "distcc" ):
         env["CXX"] = "distcc " + env["CXX"]
 
@@ -1378,7 +1379,7 @@ def doConfigure(myenv):
 
         conf.env.Append(CPPDEFINES=[("BOOST_THREAD_VERSION", "2")])
 
-        # Note that on Windows with using-system-boost builds, the following 
+        # Note that on Windows with using-system-boost builds, the following
         # FindSysLibDep calls do nothing useful (but nothing problematic either)
         for b in boostLibs:
             boostlib = "boost_" + b
@@ -1415,10 +1416,10 @@ def doConfigure(myenv):
         env.Append( CPPDEFINES=["MONGO_SASL"] )
 
     if conf.env['MONGO_BUILD_SASL_CLIENT'] and not conf.CheckLibWithHeader(
-            "sasl2", 
-            ["stddef.h","sasl/sasl.h"], 
-            "C", 
-            "sasl_version_info(0, 0, 0, 0, 0, 0);", 
+            "sasl2",
+            ["stddef.h","sasl/sasl.h"],
+            "C",
+            "sasl_version_info(0, 0, 0, 0, 0, 0);",
             autoadd=False ):
         Exit(1)
 
@@ -1482,7 +1483,7 @@ def doStyling( env , target , source ):
         print( "astyle 2.x needed, found:" + res )
         Exit(-1)
 
-    files = utils.getAllSourceFiles() 
+    files = utils.getAllSourceFiles()
     files = filter( lambda x: not x.endswith( ".c" ) , files )
 
     cmd = "astyle --options=mongo_astyle " + " ".join( files )

src/mongo/bson/bson-inl.h

@@ -1003,6 +1003,13 @@ namespace mongo {
         return true;
     }
 
+    template<> inline bool BSONElement::coerce<long long>( long long* out ) const {
+        if ( !isNumber() )
+            return false;
+        *out = numberLong();
+        return true;
+    }
+
     template<> inline bool BSONElement::coerce<double>( double* out ) const {
         if ( !isNumber() )
             return false;

src/mongo/bson/bson_validate.cpp

@@ -17,6 +17,7 @@
 
 #include <cstring>
 #include <deque>
+#include <limits>
 
 #include "mongo/bson/bson_validate.h"
 #include "mongo/bson/oid.h"
@@ -78,6 +79,11 @@ namespace mongo {
                 if ( !readNumber<int>( &sz ) )
                     return makeError("invalid bson", _idElem);
 
+                if ( sz <= 0 ) {
+                    // must have NULL at the very least
+                    return makeError("invalid bson", _idElem);
+                }
+
                 if ( out ) {
                     *out = StringData( _buffer + _position, sz );
                 }
@@ -237,6 +243,8 @@ namespace mongo {
                 int sz;
                 if ( !buffer->readNumber<int>( &sz ) )
                     return makeError("invalid bson", idElem);
+                if ( sz < 0 || sz == std::numeric_limits<int>::max() )
+                    return makeError("invalid size in bson", idElem);
                 if ( !buffer->skip( 1 + sz ) )
                     return makeError("invalid bson", idElem);
                 return Status::OK();

src/mongo/bson/bson_validate_test.cpp

@@ -273,4 +273,20 @@ namespace {
         ASSERT_NOT_OK(status);
         ASSERT_EQUALS(status.reason(), "not null terminated string in object with unknown _id");
     }
+
+    TEST(BSONValidateFast, StringHasSomething) {
+        BufBuilder bb;
+        BSONObjBuilder ob(bb);
+        bb.appendChar(String);
+        bb.appendStr("x", /*withNUL*/true);
+        bb.appendNum(0);
+        const BSONObj x = ob.done();
+        ASSERT_EQUALS(5 // overhead
+                      + 1 // type
+                      + 2 // name
+                      + 4 // size
+                      , x.objsize());
+        ASSERT_NOT_OK(validateBSON(x.objdata(), x.objsize()));
+    }
+
 }

src/mongo/bson/bsonobjbuilder.h

@@ -55,18 +55,27 @@ namespace mongo {
         BSONObjBuilder(int initsize=512) : _b(_buf), _buf(initsize + sizeof(unsigned)), _offset( sizeof(unsigned) ), _s( this ) , _tracker(0) , _doneCalled(false) {
             _b.appendNum((unsigned)0); // ref-count
             _b.skip(4); /*leave room for size field and ref-count*/
+
+            // Reserve space for the EOO byte. This means _done() can't fail.
+            _b.reserveBytes(1);
         }
 
         /** @param baseBuilder construct a BSONObjBuilder using an existing BufBuilder
          *  This is for more efficient adding of subobjects/arrays. See docs for subobjStart for example.
          */
         BSONObjBuilder( BufBuilder &baseBuilder ) : _b( baseBuilder ), _buf( 0 ), _offset( baseBuilder.len() ), _s( this ) , _tracker(0) , _doneCalled(false) {
             _b.skip( 4 );
+
+            // Reserve space for the EOO byte. This means _done() can't fail.
+            _b.reserveBytes(1);
         }
 
         BSONObjBuilder( const BSONSizeTracker & tracker ) : _b(_buf) , _buf(tracker.getSize() + sizeof(unsigned) ), _offset( sizeof(unsigned) ), _s( this ) , _tracker( (BSONSizeTracker*)(&tracker) ) , _doneCalled(false) {
             _b.appendNum((unsigned)0); // ref-count
             _b.skip(4);
+
+            // Reserve space for the EOO byte. This means _done() can't fail.
+            _b.reserveBytes(1);
         }
 
         ~BSONObjBuilder() {
@@ -600,6 +609,7 @@ namespace mongo {
         BSONObj asTempObj() {
             BSONObj temp(_done());
             _b.setlen(_b.len()-1); //next append should overwrite the EOO
+            _b.reserveBytes(1); // Rereserve room for the real EOO
             _doneCalled = false;
             return temp;
         }
@@ -681,8 +691,14 @@ namespace mongo {
                 return _b.buf() + _offset;
 
             _doneCalled = true;
+
+            // TODO remove this or find some way to prevent it from failing. Since this is intended
+            // for use with BSON() literal queries, it is less likely to result in oversized BSON.
             _s.endField();
+
+            _b.claimReservedBytes(1); // Prevents adding EOO from failing.
             _b.appendNum((char) EOO);
+
             char *data = _b.buf() + _offset;
             int size = _b.len() - _offset;
             *((int*)data) = size;

src/mongo/bson/util/builder.h

@@ -110,6 +110,7 @@ namespace mongo {
                 data = 0;
             }
             l = 0;
+            reservedBytes = 0;
         }
         ~_BufBuilder() { kill(); }
 
@@ -122,9 +123,11 @@ namespace mongo {
 
         void reset() {
             l = 0;
+            reservedBytes = 0;
         }
         void reset( int maxSize ) {
             l = 0;
+            reservedBytes = 0;
             if ( maxSize && size > maxSize ) {
                 al.Free(data);
                 data = (char*)al.Malloc(maxSize);
@@ -201,19 +204,43 @@ namespace mongo {
         inline char* grow(int by) {
             int oldlen = l;
             int newLen = l + by;
-            if ( newLen > size ) {
-                grow_reallocate(newLen);
+            int minSize = newLen + reservedBytes;
+            if ( minSize > size ) {
+                grow_reallocate(minSize);
             }
             l = newLen;
             return data + oldlen;
         }
 
+        /**
+         * Reserve room for some number of bytes to be claimed at a later time.
+         */
+        void reserveBytes(int bytes) {
+            int minSize = l + reservedBytes + bytes;
+            if (minSize > size)
+                grow_reallocate(minSize);
+
+            // This must happen *after* any attempt to grow.
+            reservedBytes += bytes;
+        }
+
+        /**
+         * Claim an earlier reservation of some number of bytes. These bytes must already have been
+         * reserved. Appends of up to this many bytes immediately following a claim are
+         * guaranteed to succeed without a need to reallocate.
+         */
+        void claimReservedBytes(int bytes) {
+            invariant(reservedBytes >= bytes);
+            reservedBytes -= bytes;
+        }
+
     private:
         /* "slow" portion of 'grow()'  */
-        void NOINLINE_DECL grow_reallocate(int newLen) {
+        void NOINLINE_DECL grow_reallocate(int minSize) {
             int a = 64;
-            while( a < newLen ) 
+            while (a < minSize)
                 a = a * 2;
+
             if ( a > BufferMaxSize ) {
                 std::stringstream ss;
                 ss << "BufBuilder attempted to grow() to " << a << " bytes, past the 64MB limit.";
@@ -228,6 +255,7 @@ namespace mongo {
         char *data;
         int l;
         int size;
+        int reservedBytes; // eagerly grow_reallocate to keep this many bytes of spare room.
 
         friend class StringBuilderImpl<Allocator>;
     };