Merge branch 'master' into upstream-gitpython

Author: DaveLak

infra/base-images/base-builder/jcc/jcc.go

@@ -357,7 +357,7 @@ func WriteStdErrOut(outstr string, errstr string) {
 	// Prints |outstr| to stdout, prints |errstr| to stderr, and saves |errstr| to err.log.
 	fmt.Print(outstr)
 	fmt.Fprint(os.Stderr, errstr)
-	AppendStringToFile("/out/err.log", errstr)
+	AppendStringToFile("/workspace/err.log", errstr)
 }
 
 func main() {

infra/base-images/base-clang/Dockerfile

@@ -36,7 +36,7 @@ RUN apt-get update && apt-get install -y wget sudo && \
 RUN apt-get update && apt-get install -y git && \
     git clone https://github.com/ossf/fuzz-introspector.git fuzz-introspector && \
     cd fuzz-introspector && \
-    git checkout bb419272223f251599a5f9c7581073e23a487369 && \
+    git checkout cfb5266a4c45cbec8663bb1b215c7fd326c60901 && \
     git submodule init && \
     git submodule update && \
     apt-get autoremove --purge -y git && \

infra/build/functions/target_experiment.py

@@ -28,7 +28,7 @@
 JCC_DIR = '/usr/local/bin'
 
 
-def run_experiment(project_name, target_name, args, output_path,
+def run_experiment(project_name, target_name, args, output_path, errlog_path,
                    build_output_path, upload_corpus_path, upload_coverage_path,
                    experiment_name, upload_reproducer_path):
   config = build_project.Config(testing=True,
@@ -72,6 +72,7 @@ def run_experiment(project_name, target_name, args, output_path,
 
   build = build_project.Build('libfuzzer', 'address', 'x86_64')
   local_output_path = '/workspace/output.log'
+  local_jcc_err_path = '/workspace/err.log'  # From jcc.go:360.
   local_corpus_path_base = '/workspace/corpus'
   local_corpus_path = os.path.join(local_corpus_path_base, target_name)
   default_target_path = os.path.join(build.out, target_name)
@@ -81,6 +82,24 @@ def run_experiment(project_name, target_name, args, output_path,
   local_stacktrace_path = os.path.join(build.out, 'stacktrace/')
   fuzzer_args = ' '.join(args + [f'-artifact_prefix={local_artifact_path}'])
 
+  # Upload JCC's err.log.
+  if errlog_path:
+    compile_step_index = -1
+    for i, step in enumerate(steps):
+      step_args = step.get('args', [])
+      if '&& compile' in ' '.join(step_args):
+        compile_step_index = i
+        break
+    if compile_step_index == -1:
+      print('Cannot find compile step.')
+    else:
+      # Insert the upload step right after compile step.
+      upload_jcc_err_step = {
+          'name': 'gcr.io/cloud-builders/gsutil',
+          'args': ['cp', local_jcc_err_path, errlog_path]
+      }
+      steps.insert(compile_step_index + 1, upload_jcc_err_step)
+
   env = build_project.get_env(project_yaml['language'], build)
   env.append('RUN_FUZZER_MODE=batch')
   env.append('CORPUS_DIR=' + local_corpus_path)
@@ -268,6 +287,10 @@ def main():
   parser.add_argument('--upload_build_log',
                       required=True,
                       help='GCS build log location.')
+  parser.add_argument('--upload_err_log',
+                      required=False,
+                      default='',
+                      help='GCS JCC error log location.')
   parser.add_argument('--upload_output_log',
                       required=True,
                       help='GCS log location.')
@@ -287,7 +310,7 @@ def main():
   args = parser.parse_args()
 
   run_experiment(args.project, args.target, args.args, args.upload_output_log,
-                 args.upload_build_log, args.upload_corpus,
+                 args.upload_err_log, args.upload_build_log, args.upload_corpus,
                  args.upload_coverage, args.experiment_name,
                  args.upload_reproducer)
 

projects/apache-poi/Dockerfile

@@ -42,7 +42,10 @@ WORKDIR ${SRC}
 RUN git clone --depth 1 https://github.com/apache/poi.git
 
 # install packages required for font-handling and other code in java.awt.*
-RUN apt-get install -y libxext6 libx11-6 libxrender1 libxtst6 libxi6 libxcb1 libxau6 libxdmcp6
+RUN apt-get install -y libxext6 libx11-6 libxrender1 libxtst6 libxi6 libxcb1 libxau6 libxdmcp6 \
+&& apt-get clean autoclean \
+&& apt-get autoremove --yes \
+&& rm -rf /var/lib/{apt,dpkg,cache,log}/
 
 ADD pom.xml build.sh ${SRC}/
 ADD src/ ${SRC}/src/

projects/apache-poi/src/main/java/org/apache/poi/POIFuzzer.java

@@ -124,8 +124,16 @@ public static void checkExtractor(byte[] input) {
 	public static void checkExtractor(POITextExtractor extractor) throws IOException {
 		extractor.getDocument();
 		extractor.getFilesystem();
-		extractor.getMetadataTextExtractor();
-		extractor.getText();
+		try {
+			extractor.getMetadataTextExtractor();
+		} catch (IllegalStateException e) {
+			// can happen here
+		}
+		try {
+			extractor.getText();
+		} catch (OpenXML4JRuntimeException e) {
+			// can happen here
+		}
 
 		if (extractor instanceof POIOLE2TextExtractor) {
 			POIOLE2TextExtractor ole2Extractor = (POIOLE2TextExtractor) extractor;

projects/apache-poi/src/main/java/org/apache/poi/POIVisioFuzzer.java

@@ -38,7 +38,7 @@ public static void fuzzerTestOneInput(byte[] input) {
 			visio.write(NullOutputStream.INSTANCE);
 		} catch (IOException | POIXMLException |
 				 BufferUnderflowException | RecordFormatException | OpenXML4JRuntimeException |
-				 IllegalArgumentException | IndexOutOfBoundsException e) {
+				 IllegalArgumentException | IndexOutOfBoundsException | IllegalStateException e) {
 			// expected here
 		}
 

projects/hsqldb/Dockerfile

@@ -16,14 +16,14 @@
 
 FROM gcr.io/oss-fuzz-base/base-builder-jvm
 
-RUN curl -L https://dlcdn.apache.org//ant/binaries/apache-ant-1.10.12-bin.zip -o ant.zip && \
+RUN curl -L https://dlcdn.apache.org//ant/binaries/apache-ant-1.10.14-bin.zip -o ant.zip && \
     unzip ant.zip -d $SRC/ant && \
     rm -rf ant.zip
 
-ENV ANT $SRC/ant/apache-ant-1.10.12/bin/ant
+ENV ANT $SRC/ant/apache-ant-1.10.14/bin/ant
 
 RUN svn checkout https://svn.code.sf.net/p/hsqldb/svn/base/trunk hsqldb-svn
 
 COPY build.sh $SRC/
 COPY *.java $SRC/
-WORKDIR $SRC/hsqldb
+WORKDIR $SRC/hsqldb

projects/libyaml/project.yaml

@@ -1,19 +1,17 @@
 homepage: https://github.com/yaml/libyaml
 language: c++
-primary_contact: "[email protected]"
+primary_contact: [email protected]
 auto_ccs:
-  - "[email protected]"
-  - "[email protected]"
-  - "[email protected]"
+- [email protected]
 fuzzing_engines:
-  - libfuzzer
-  - afl
-  - honggfuzz
+- libfuzzer
+- afl
+- honggfuzz
 sanitizers:
-  - address
-  - memory
-  - undefined
+- address
+- memory
+- undefined
 architectures:
-  - x86_64
-  - i386
-main_repo: 'https://github.com/yaml/libyaml'
+- x86_64
+- i386
+main_repo: https://github.com/yaml/libyaml

projects/lua/build.sh

@@ -65,6 +65,12 @@ esac
 
 export LSAN_OPTIONS="verbosity=1:log_threads=1"
 
+# Workaround for a LeakSanitizer crashes,
+# see https://github.com/google/oss-fuzz/issues/11798.
+if [ "$ARCHITECTURE" = "aarch64" ]; then
+    export ASAN_OPTIONS=detect_leaks=0
+fi
+
 : ${LD:="${CXX}"}
 : ${LDFLAGS:="${CXXFLAGS}"}  # to make sure we link with sanitizer runtime