Sync from Piper @365872496

PROTOBUF_SYNC_PIPER

Author: haberman

CHANGES.txt

@@ -9,9 +9,32 @@ Unreleased Changes (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript)
   * For iterator-pair function parameter types, take both iterators by value.
   * Code-space savings and perhaps some modest performance improvements in
     RepeatedPtrField.
+  * Eliminate nullptr check from every tag parse.
+  * Remove unused _$name$_cached_byte_size_ fields.
+  * Serialize extension ranges together when not broken by a proto field in the
+    middle.
+  * Do out-of-line allocation and deallocation of string object in ArenaString.
+  * Streamline ParseContext::ParseMessage<T> to avoid code bloat and improve
+    performance.
+  * New member functions RepeatedField::Assign, RepeatedPtrField::{Add, Assign}.
+  * Fix undefined behavior warning due to innocuous uninitialization of value
+    on an error path.
+  * Avoid expensive inlined code space for encoding message length for messages
+    >= 128 bytes and instead do a procedure call to a shared out-of-line routine.
 
   Java:
-  * Exceptions thrown while reading from an InputStream in parseFrom are now included as causes.
+  * Exceptions thrown while reading from an InputStream in parseFrom are now
+    included as causes.
+  * Support potentially more efficient proto parsing from RopeByteStrings.
+  * Clarify runtime of ByteString.Output.toStringBuffer().
+
+  Python
+  * Fixed a bug in text format where a trailing colon was printed for repeated field.
+  * When TextFormat encounters a duplicate message map key, replace the current
+    one instead of merging.
+
+  JavaScript
+  * Make Any.pack() chainable.
 
 2021-03-10 version 3.15.6 (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript)
 

Protobuf-C++.podspec

@@ -1,6 +1,6 @@
 Pod::Spec.new do |s|
   s.name     = 'Protobuf-C++'
-  s.version  = '3.15.2'
+  s.version  = '3.15.6'
   s.summary  = 'Protocol Buffers v3 runtime library for C++.'
   s.homepage = 'https://github.com/google/protobuf'
   s.license  = '3-Clause BSD License'

cmake/CMakeLists.txt

@@ -45,6 +45,7 @@ option(protobuf_BUILD_CONFORMANCE "Build conformance tests" OFF)
 option(protobuf_BUILD_EXAMPLES "Build examples" OFF)
 option(protobuf_BUILD_PROTOC_BINARIES "Build libprotoc and protoc compiler" ON)
 option(protobuf_BUILD_LIBPROTOC "Build libprotoc" OFF)
+option(protobuf_DISABLE_RTTI "Remove runtime type information in the binaries" OFF)
 if (BUILD_SHARED_LIBS)
   set(protobuf_BUILD_SHARED_LIBS_DEFAULT ON)
 else (BUILD_SHARED_LIBS)
@@ -117,6 +118,10 @@ endif()
 
 add_definitions(-DGOOGLE_PROTOBUF_CMAKE_BUILD)
 
+if (protobuf_DISABLE_RTTI)
+  add_definitions(-DGOOGLE_PROTOBUF_NO_RTTI=1)
+endif()
+
 find_package(Threads REQUIRED)
 if (CMAKE_USE_PTHREADS_INIT)
   add_definitions(-DHAVE_PTHREAD)

cmake/install.cmake

@@ -30,7 +30,9 @@ endforeach()
 
 if (protobuf_BUILD_PROTOC_BINARIES)
   install(TARGETS protoc EXPORT protobuf-targets
-    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT protoc)
+    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+    BUNDLE DESTINATION ${CMAKE_INSTALL_BINDIR}
+    COMPONENT protoc)
   if (UNIX AND NOT APPLE)
     set_property(TARGET protoc
       PROPERTY INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}")

cmake/protobuf-config.cmake.in

@@ -15,7 +15,7 @@ function(protobuf_generate)
   if(COMMAND target_sources)
     list(APPEND _singleargs TARGET)
   endif()
-  set(_multiargs PROTOS IMPORT_DIRS GENERATE_EXTENSIONS)
+  set(_multiargs PROTOS IMPORT_DIRS GENERATE_EXTENSIONS PROTOC_OPTIONS)
 
   cmake_parse_arguments(protobuf_generate "${_options}" "${_singleargs}" "${_multiargs}" "${ARGN}")
 
@@ -130,9 +130,9 @@ function(protobuf_generate)
     add_custom_command(
       OUTPUT ${_generated_srcs}
       COMMAND  protobuf::protoc
-      ARGS --${protobuf_generate_LANGUAGE}_out ${_dll_export_decl}${protobuf_generate_PROTOC_OUT_DIR} ${_plugin} ${_protobuf_include_path} ${_abs_file}
+      ARGS ${protobuf_generate_PROTOC_OPTIONS} --${protobuf_generate_LANGUAGE}_out ${_dll_export_decl}${protobuf_generate_PROTOC_OUT_DIR} ${_plugin} ${_protobuf_include_path} ${_abs_file}
       DEPENDS ${_abs_file} protobuf::protoc
-      COMMENT "Running ${protobuf_generate_LANGUAGE} protocol buffer compiler on ${_proto}"
+      COMMENT "Running ${protobuf_generate_LANGUAGE} protocol buffer compiler on ${_proto}. Custom options: ${protobuf_generate_PROTOC_OPTIONS}"
       VERBATIM )
   endforeach()
 

configure.ac

@@ -17,7 +17,7 @@ AC_PREREQ(2.59)
 # In the SVN trunk, the version should always be the next anticipated release
 # version with the "-pre" suffix.  (We used to use "-SNAPSHOT" but this pushed
 # the size of one file name in the dist tarfile over the 99-char limit.)
-AC_INIT([Protocol Buffers],[3.15.2],[[email protected]],[protobuf])
+AC_INIT([Protocol Buffers],[3.15.6],[[email protected]],[protobuf])
 
 AM_MAINTAINER_MODE([enable])
 
@@ -223,6 +223,19 @@ case "$target_os" in
 esac
 AM_CONDITIONAL([OBJC_CONFORMANCE_TEST], [test $OBJC_CONFORMANCE_TEST = 1])
 
+AC_MSG_CHECKING(whether -llog is needed)
+ANDROID_TEST=no
+case "$target_os" in
+  *android*)
+    ANDROID_TEST=yes
+    ;;
+esac
+AC_MSG_RESULT($ANDROID_TEST)
+if test "x$ANDROID_TEST" = xyes; then
+  LIBLOG_LIBS="-llog"
+fi
+AC_SUBST([LIBLOG_LIBS])
+
 # HACK:  Make gmock's configure script pick up our copy of CFLAGS and CXXFLAGS,
 #   since the flags added by ACX_CHECK_SUNCC must be used when compiling gmock
 #   too.

conformance/conformance_cpp.cc

@@ -134,7 +134,7 @@ void DoTest(const ConformanceRequest& request, ConformanceResponse* response) {
                              &proto_binary, options);
       if (!status.ok()) {
         response->set_parse_error(string("Parse error: ") +
-                                  std::string(status.error_message()));
+                                  std::string(status.message()));
         return;
       }
 
@@ -189,7 +189,7 @@ void DoTest(const ConformanceRequest& request, ConformanceResponse* response) {
       if (!status.ok()) {
         response->set_serialize_error(
             string("Failed to serialize JSON output: ") +
-            std::string(status.error_message()));
+            std::string(status.message()));
         return;
       }
       break;

java/core/BUILD

@@ -85,6 +85,7 @@ LITE_SRCS = [
     "src/main/java/com/google/protobuf/UnknownFieldSetLite.java",
     "src/main/java/com/google/protobuf/UnknownFieldSetLiteSchema.java",
     "src/main/java/com/google/protobuf/UnmodifiableLazyStringList.java",
+    "src/main/java/com/google/protobuf/UnsafeByteOperations.java",
     "src/main/java/com/google/protobuf/UnsafeUtil.java",
     "src/main/java/com/google/protobuf/Utf8.java",
     "src/main/java/com/google/protobuf/WireFormat.java",

java/core/src/main/java/com/google/protobuf/CodedInputStream.java

@@ -3523,9 +3523,25 @@ public ByteString readBytes() throws IOException {
           return ByteString.wrap(bytes);
         }
       } else if (size > 0 && size <= remaining()) {
-        byte[] temp = new byte[size];
-        readRawBytesTo(temp, 0, size);
-        return ByteString.wrap(temp);
+        if (immutable && enableAliasing) {
+          ArrayList<ByteString> byteStrings = new ArrayList<>();
+          int l = size;
+          while (l > 0) {
+            if (currentRemaining() == 0) {
+              getNextByteBuffer();
+            }
+            int bytesToCopy = Math.min(l, (int) currentRemaining());
+            int idx = (int) (currentByteBufferPos - currentAddress);
+            byteStrings.add(ByteString.wrap(slice(idx, idx + bytesToCopy)));
+            l -= bytesToCopy;
+            currentByteBufferPos += bytesToCopy;
+          }
+          return ByteString.copyFrom(byteStrings);
+        } else {
+          byte[] temp = new byte[size];
+          readRawBytesTo(temp, 0, size);
+          return ByteString.wrap(temp);
+        }
       }
 
       if (size == 0) {

java/core/src/main/java/com/google/protobuf/InvalidProtocolBufferException.java

@@ -78,7 +78,7 @@ void setThrownFromInputStream() {
     /* This write can be racy if the same exception is stored and then thrown by multiple custom
      * InputStreams on different threads. But since it only ever moves from false->true, there's no
      * problem. A thread checking this condition after catching this exception from a delegate
-     * stram of CodedInputStream is guaranteed to always observe true, because a write on the same
+     * stream of CodedInputStream is guaranteed to always observe true, because a write on the same
      * thread set the value when the exception left the delegate. A thread checking the same
      * condition with an exception created by CodedInputStream is guaranteed to always see false,
      * because the exception has not been exposed to any code that could publish it to other threads

java/core/src/test/java/com/google/protobuf/CodedInputStreamTest.java

@@ -1142,6 +1142,39 @@ public void testReadByteBufferAliasing() throws Exception {
     }
   }
 
+  public void testIterableByteBufferInputStreamReadBytesWithAlias() throws Exception {
+    ByteArrayOutputStream byteArrayStream = new ByteArrayOutputStream();
+    CodedOutputStream output = CodedOutputStream.newInstance(byteArrayStream);
+    // A bytes field large enough that won't fit into the default block buffer.
+    // 4.5 is to test the case where the total size of input is not aligned with DEFAULT_BLOCK_SIZE.
+    final int bytesLength = DEFAULT_BLOCK_SIZE * 4 + (DEFAULT_BLOCK_SIZE / 2);
+    byte[] bytes = new byte[bytesLength];
+    for (int i = 0; i < bytesLength; i++) {
+      bytes[i] = (byte) (i % 256);
+    }
+    output.writeByteArrayNoTag(bytes);
+    output.flush();
+
+    // Input data is split into multiple ByteBuffers so that a single bytes spans across them.
+    // CodedInputStream with aliasing will decode it as a consequent rope by wrapping ByteBuffers.
+    byte[] data = byteArrayStream.toByteArray();
+    ArrayList<ByteBuffer> input = new ArrayList<>();
+    for (int i = 0; i < data.length; i += DEFAULT_BLOCK_SIZE) {
+      int rl = Math.min(DEFAULT_BLOCK_SIZE, data.length - i);
+      ByteBuffer rb = ByteBuffer.allocateDirect(rl);
+      rb.put(data, i, rl);
+      rb.flip();
+      input.add(rb);
+    }
+    final CodedInputStream inputStream = CodedInputStream.newInstance(input, true);
+    inputStream.enableAliasing(true);
+
+    ByteString result = inputStream.readBytes();
+    for (int i = 0; i < bytesLength; i++) {
+      assertEquals((byte) (i % 256), result.byteAt(i));
+    }
+  }
+
   public void testCompatibleTypes() throws Exception {
     long data = 0x100000000L;
     Int64Message message = Int64Message.newBuilder().setData(data).build();
@@ -1196,7 +1229,7 @@ public void testSkipPastEndOfByteArrayInput() throws Exception {
       // Expected
     }
   }
-  
+
   public void testMaliciousInputStream() throws Exception {
     ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
     CodedOutputStream codedOutputStream = CodedOutputStream.newInstance(outputStream);
@@ -1210,28 +1243,28 @@ public synchronized int read(byte[] b, int off, int len) {
         return super.read(b, off, len);
       }
     };
-    
+
     // test ByteString
-    
+
     CodedInputStream codedInputStream = CodedInputStream.newInstance(inputStream, 1);
     ByteString byteString = codedInputStream.readBytes();
     assertEquals(0x0, byteString.byteAt(0));
     maliciousCapture.get(1)[0] = 0x9;
     assertEquals(0x0, byteString.byteAt(0));
-    
+
     // test ByteBuffer
-    
+
     inputStream.reset();
     maliciousCapture.clear();
     codedInputStream = CodedInputStream.newInstance(inputStream, 1);
     ByteBuffer byteBuffer = codedInputStream.readByteBuffer();
     assertEquals(0x0, byteBuffer.get(0));
     maliciousCapture.get(1)[0] = 0x9;
     assertEquals(0x0, byteBuffer.get(0));
-    
+
 
     // test byte[]
-    
+
     inputStream.reset();
     maliciousCapture.clear();
     codedInputStream = CodedInputStream.newInstance(inputStream, 1);
@@ -1241,7 +1274,7 @@ public synchronized int read(byte[] b, int off, int len) {
     assertEquals(0x9, byteArray[0]); // MODIFICATION! Should we fix?
 
     // test rawBytes
-    
+
     inputStream.reset();
     maliciousCapture.clear();
     codedInputStream = CodedInputStream.newInstance(inputStream, 1);

java/lite/pom.xml

@@ -173,6 +173,7 @@
                     <include>UnknownFieldSetLite.java</include>
                     <include>UnknownFieldSetLiteSchema.java</include>
                     <include>UnmodifiableLazyStringList.java</include>
+                    <include>UnsafeByteOperations.java</include>
                     <include>UnsafeUtil.java</include>
                     <include>Utf8.java</include>
                     <include>WireFormat.java</include>

java/util/src/main/java/com/google/protobuf/util/Durations.java

@@ -149,6 +149,12 @@ public static boolean isNegative(Duration duration) {
     return (duration.getSeconds() == 0) ? duration.getNanos() < 0 : duration.getSeconds() < 0;
   }
 
+  /** Returns whether the given {@link Duration} is positive or not. */
+  public static boolean isPositive(Duration duration) {
+    checkValid(duration);
+    return !isNegative(duration) && !duration.equals(ZERO);
+  }
+
   /**
    * Ensures that the given {@link Duration} is not negative.
    *
@@ -157,7 +163,6 @@ public static boolean isNegative(Duration duration) {
    */
   @CanIgnoreReturnValue
   public static Duration checkNotNegative(Duration duration) {
-    checkValid(duration);
     checkArgument(!isNegative(duration), "duration (%s) must not be negative", toString(duration));
     return duration;
   }
@@ -170,11 +175,7 @@ public static Duration checkNotNegative(Duration duration) {
    */
   @CanIgnoreReturnValue
   public static Duration checkPositive(Duration duration) {
-    checkValid(duration);
-    checkArgument(
-        !isNegative(duration) && !duration.equals(ZERO),
-        "duration (%s) must be positive",
-        toString(duration));
+    checkArgument(isPositive(duration), "duration (%s) must be positive", toString(duration));
     return duration;
   }
 

js/package.json

@@ -1,6 +1,6 @@
 {
   "name": "google-protobuf",
-  "version": "3.15.2",
+  "version": "3.15.6",
   "description": "Protocol Buffers for JavaScript",
   "main": "google-protobuf.js",
   "files": [

kokoro/release/python/linux/build_artifacts.sh

@@ -49,9 +49,31 @@ build_artifact_version() {
   sudo rm -rf $REPO_DIR
 }
 
+build_crosscompiled_aarch64_artifact_version() {
+  # crosscompilation is only supported with the dockcross manylinux2014 image
+  DOCKER_IMAGE=dockcross/manylinux2014-aarch64
+  PLAT=aarch64
+
+  # TODO(jtatermusch): currently when crosscompiling, "auditwheel repair" will be disabled
+  # since auditwheel doesn't work for crosscomiled wheels.
+  build_artifact_version $@
+}
+
 build_artifact_version 2.7
 build_artifact_version 3.5
 build_artifact_version 3.6
 build_artifact_version 3.7
 build_artifact_version 3.8
 build_artifact_version 3.9
+
+build_crosscompiled_aarch64_artifact_version 3.7
+build_crosscompiled_aarch64_artifact_version 3.8
+build_crosscompiled_aarch64_artifact_version 3.9
+
+# Put the aarch64 manylinux wheels under the "unofficial" subdirectory.
+# Only wheels directly under the artifacts/ directory will be published
+# to PyPI as part of the protobuf release process.
+# TODO(jtattermusch): include aarch64 wheels in the release
+# once they are sufficiently tested.
+mkdir -p $ARTIFACT_DIR/unofficial
+mv $ARTIFACT_DIR/protobuf-*-manylinux*_aarch64.whl $ARTIFACT_DIR/unofficial