Allow user-defined Makefile to optimize compiling

.github/workflows/linux.yaml

@@ -87,3 +87,13 @@ jobs:
           ruby-version: 2.6
       - name: Check usage - Test SharedLibrary should fail
         run: ./SampleProjects/SharedLibrary/test.sh
+
+  Makefile:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 2.6
+      - name: Check usage - Demo Makefile
+        run: ./SampleProjects/Makefile/test.sh

.github/workflows/macos.yaml

@@ -72,3 +72,14 @@ jobs:
           ruby-version: 2.6
       - name: Check usage - Test SharedLibrary should fail
         run: ./SampleProjects/SharedLibrary/test.sh
+
+  Makefile:
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 2.6
+      - name: Check usage - Demo Makefile
+        run: ./SampleProjects/Makefile/test.sh
+

.github/workflows/windows.yaml

@@ -72,3 +72,15 @@ jobs:
           ruby-version: 2.6
       - name: Check usage - Test SharedLibrary should fail
         run: ./SampleProjects/SharedLibrary/test.sh
+
+
+  Makefile:
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 2.6
+      - name: Check usage - Demo Makefile
+        run: ./SampleProjects/Makefile/test.sh
+

CHANGELOG.md

@@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ## [Unreleased]
 ### Added
+- Support for optional Makefile to optimize compile performance
 
 ### Changed
 

SampleProjects/Makefile/.arduino-ci.yml

@@ -0,0 +1,7 @@
+unittest:
+  platforms:
+    - mega2560
+
+compile:
+  platforms:
+    - mega2560

SampleProjects/Makefile/.gitignore

@@ -0,0 +1 @@
+.bundle

SampleProjects/Makefile/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gem 'arduino_ci', path: '../../'

SampleProjects/Makefile/Makefile

@@ -0,0 +1,61 @@
+# ARDUINO_CI should point to cpp/ and be provided as a shell environment variable
+LIBRARIES := $(shell bundle exec arduino_library_location.rb)
+SRC=$(LIBRARIES)/Makefile/src
+TEST=$(LIBRARIES)/Makefile/test
+BIN=$(LIBRARIES)/Makefile/.arduino_ci
+
+FLAGS=-std=c++0x \
+	-Wno-deprecated-declarations \
+  -DARDUINO=100 \
+  -fPIC \
+  -g \
+  -O1 \
+  -fno-omit-frame-pointer \
+  -fno-optimize-sibling-calls \
+  -fsanitize=address \
+  -Wno-unknown-attributes \
+  -Wno-address-of-packed-member \
+  -D__AVR__ \
+  -D__AVR_ATmega2560__ \
+  -DARDUINO_ARCH_AVR \
+  -DARDUINO_AVR_MEGA2560
+
+INCLUDE=-I$(ARDUINO_CI)/arduino \
+  -I$(ARDUINO_CI)/unittest \
+  -I$(LIBRARIES)/Makefile/src \
+
+GPP_TEST=g++ $(FLAGS) -L$(BIN) $(INCLUDE)
+
+.PHONY : all
+all : $(BIN)/test.cpp.bin
+
+
+$(BIN)/test.cpp.bin: $(BIN)/libarduino.so $(TEST)/test.cpp
+	$(GPP_TEST) -o $(BIN)/test.cpp.bin $(TEST)/test.cpp -larduino
+
+OBJECTS=$(BIN)/stdlib.o $(BIN)/Godmode.o $(BIN)/Arduino.o $(BIN)/ArduinoUnitTests.o \
+	$(BIN)/SharedLibrary.o 
+
+$(BIN)/libarduino.so: $(OBJECTS)
+	g++ $(FLAGS) -shared -Wl,-undefined,dynamic_lookup -L$(BIN) $(INCLUDE) \
+	-o $(BIN)/libarduino.so $(OBJECTS)
+
+$(BIN)/stdlib.o: $(ARDUINO_CI)/arduino/stdlib.cpp
+	g++ -c $(FLAGS) $(INCLUDE) -o $(BIN)/stdlib.o $(ARDUINO_CI)/arduino/stdlib.cpp
+
+$(BIN)/Godmode.o: $(ARDUINO_CI)/arduino/Godmode.cpp
+	g++ -c $(FLAGS) $(INCLUDE) -o $(BIN)/Godmode.o $(ARDUINO_CI)/arduino/Godmode.cpp
+
+$(BIN)/Arduino.o: $(ARDUINO_CI)/arduino/Arduino.cpp
+	g++ -c $(FLAGS) $(INCLUDE) -o $(BIN)/Arduino.o $(ARDUINO_CI)/arduino/Arduino.cpp
+
+$(BIN)/ArduinoUnitTests.o: $(ARDUINO_CI)/unittest/ArduinoUnitTests.cpp
+	g++ -c $(FLAGS) $(INCLUDE) -o $(BIN)/ArduinoUnitTests.o $(ARDUINO_CI)/unittest/ArduinoUnitTests.cpp
+
+$(BIN)/SharedLibrary.o: $(SRC)/SharedLibrary.cpp
+	g++ -c $(FLAGS) $(INCLUDE) -o $(BIN)/SharedLibrary.o $(SRC)/SharedLibrary.cpp
+
+
+.PHONY: clean
+clean:
+	rm -rf $(BIN)/*

SampleProjects/Makefile/README.md

@@ -0,0 +1,4 @@
+# Makefile
+
+This sample project shows that you can create your own Makefile and bypass the (much slower!) Arduino CI process.
+Of course, you are then responsible for managing the list of files and prerequisites yourself!

SampleProjects/Makefile/library.properties

@@ -0,0 +1,10 @@
+name=Makefile
+version=0.1.0
+author=James Foster <[email protected]>
+maintainer=James Foster <[email protected]>
+sentence=Show how to use a Makefile to shorten compile time
+paragraph=Show how to use a Makefile to shorten compile time
+category=Other
+url=https://github.com/Arduino-CI/arduino_ci/SampleProjects/Makefile
+architectures=avr,esp8266
+includes=Makefile.h

SampleProjects/Makefile/src/SharedLibrary.cpp

@@ -0,0 +1,5 @@
+#include "SharedLibrary.h"
+
+int main() {
+  return 0;
+}

SampleProjects/Makefile/src/SharedLibrary.h

@@ -0,0 +1,3 @@
+#pragma once
+
+#include <Arduino.h>

SampleProjects/Makefile/test.sh

@@ -0,0 +1,5 @@
+g++ -v
+cd SampleProjects/Makefile
+bundle install
+bundle exec ensure_arduino_installation.rb
+bundle exec arduino_ci.rb --skip-examples-compilation

SampleProjects/Makefile/test/test.cpp

@@ -0,0 +1,13 @@
+/*
+cd SampleProjects/Makefile
+bundle config --local path vendor/bundle
+bundle install
+bundle exec arduino_ci.rb --skip-examples-compilation
+*/
+
+#include <Arduino.h>
+#include <ArduinoUnitTests.h>
+
+unittest(test) { assertEqual(true, true); }
+
+unittest_main()

exe/arduino_ci.rb

@@ -159,7 +159,7 @@ def display_files(pathname)
 # forcibly installs the dependency.  Each child dependency logs which parent requested it
 #
 # @param library_names [Array<String>] the list of libraries to install
-# @param on_behalf_of [String] the requestor of a given dependency
+# @param on_behalf_of [String] the requester of a given dependency
 # @param already_installed [Array<String>] the set of dependencies installed by previous steps
 # @return [Array<String>] The list of installed libraries
 def install_arduino_library_dependencies_h(library_names, on_behalf_of, already_installed)
@@ -456,7 +456,7 @@ def perform_unit_tests(cpp_library, file_config)
     @log.iputs
     compilers.each do |gcc_binary|
       # before compiling the tests, build a shared library of everything except the test code
-      next @log.failure_count += 1 unless build_shared_library(gcc_binary, p, config, cpp_library)
+      next unless build_shared_library(gcc_binary, p, config, cpp_library)
 
       # now build and run each test using the shared library build above
       config.allowable_unittest_files(cpp_library.test_files).each do |unittest_path|

lib/arduino_ci/cpp_library.rb

@@ -494,20 +494,24 @@ def test_args(aux_libraries, ci_gcc_config)
     # @return [Pathname] path to the compiled test executable
     def build_for_test(test_file, gcc_binary)
       executable = Pathname.new("#{BUILD_DIR}/#{test_file.basename}.bin").expand_path
-      File.delete(executable) if File.exist?(executable)
-      arg_sets = ["-std=c++0x", "-o", executable.to_s, "-L#{BUILD_DIR}", "-DARDUINO=100"]
-      if libasan?(gcc_binary)
-        arg_sets << [ # Stuff to help with dynamic memory mishandling
-          "-g", "-O1",
-          "-fno-omit-frame-pointer",
-          "-fno-optimize-sibling-calls",
-          "-fsanitize=address"
-        ]
+      if File.file?("Makefile") && ArduinoCI::Host.os != :windows
+        # we should already have everything built!
+      else
+        File.delete(executable) if File.exist?(executable)
+        arg_sets = ["-std=c++0x", "-o", executable.to_s, "-L#{BUILD_DIR}", "-DARDUINO=100"]
+        if libasan?(gcc_binary)
+          arg_sets << [ # Stuff to help with dynamic memory mishandling
+            "-g", "-O1",
+            "-fno-omit-frame-pointer",
+            "-fno-optimize-sibling-calls",
+            "-fsanitize=address"
+          ]
+        end
+        arg_sets << @test_args
+        arg_sets << [test_file.to_s, "-l#{LIBRARY_NAME}"]
+        args = arg_sets.flatten(1)
+        return nil unless run_gcc(gcc_binary, *args)
       end
-      arg_sets << @test_args
-      arg_sets << [test_file.to_s, "-l#{LIBRARY_NAME}"]
-      args = arg_sets.flatten(1)
-      return nil unless run_gcc(gcc_binary, *args)
 
       artifacts << executable
       executable
@@ -536,31 +540,39 @@ def build_shared_library(aux_libraries, gcc_binary, ci_gcc_config)
       suffix = OS.windows? ? "dll" : "so"
       full_lib_name = "#{BUILD_DIR}/lib#{LIBRARY_NAME}.#{suffix}"
       executable = Pathname.new(full_lib_name).expand_path
-      File.delete(executable) if File.exist?(executable)
-      arg_sets = ["-std=c++0x", "-shared", "-fPIC", "-Wl,-undefined,dynamic_lookup",
-                  "-o", executable.to_s, "-L#{BUILD_DIR}", "-DARDUINO=100"]
-      if libasan?(gcc_binary)
-        arg_sets << [ # Stuff to help with dynamic memory mishandling
-          "-g", "-O1",
-          "-fno-omit-frame-pointer",
-          "-fno-optimize-sibling-calls",
-          "-fsanitize=address"
-        ]
-      end
+      if File.file?("Makefile") && ArduinoCI::Host.os != :windows
+        @last_cmd = " $ make --jobs"
+        ret = Host.run_and_capture("export ARDUINO_CI=#{CI_CPP_DIR}; make --jobs")
+        @last_err = ret[:err]
+        @last_out = ret[:out]
+        return nil unless ret[:success]
+      else
+        File.delete(executable) if File.exist?(executable)
+        arg_sets = ["-std=c++0x", "-shared", "-fPIC", "-Wl,-undefined,dynamic_lookup",
+                    "-o", executable.to_s, "-L#{BUILD_DIR}", "-DARDUINO=100"]
+        if libasan?(gcc_binary)
+          arg_sets << [ # Stuff to help with dynamic memory mishandling
+            "-g", "-O1",
+            "-fno-omit-frame-pointer",
+            "-fno-optimize-sibling-calls",
+            "-fsanitize=address"
+          ]
+        end
 
-      # combine library.properties defs (if existing) with config file.
-      # TODO: as much as I'd like to rely only on the properties file(s), I think that would prevent testing 1.0-spec libs
-      # the following two take some time, so are cached when we build the shared library
-      @full_dependencies = all_arduino_library_dependencies!(aux_libraries)
-      @test_args = test_args(@full_dependencies, ci_gcc_config) # build full set of include directories to be cached for later
-
-      arg_sets << @test_args
-      arg_sets << cpp_files_arduino.map(&:to_s)  # Arduino.cpp, Godmode.cpp, and stdlib.cpp
-      arg_sets << cpp_files_unittest.map(&:to_s) # ArduinoUnitTests.cpp
-      arg_sets << cpp_files.map(&:to_s) # CPP files for the primary application library under test
-      arg_sets << cpp_files_libraries(@full_dependencies).map(&:to_s) # CPP files for all the libraries we depend on
-      args = arg_sets.flatten(1)
-      return nil unless run_gcc(gcc_binary, *args)
+        # combine library.properties defs (if existing) with config file.
+        # TODO: as much as I'd like to rely only on the properties file(s), I think that would prevent testing 1.0-spec libs
+        # the following two take some time, so are cached when we build the shared library
+        @full_dependencies = all_arduino_library_dependencies!(aux_libraries)
+        @test_args = test_args(@full_dependencies, ci_gcc_config) # build full set of include directories to be cached for later
+
+        arg_sets << @test_args
+        arg_sets << cpp_files_arduino.map(&:to_s)  # Arduino.cpp, Godmode.cpp, and stdlib.cpp
+        arg_sets << cpp_files_unittest.map(&:to_s) # ArduinoUnitTests.cpp
+        arg_sets << cpp_files.map(&:to_s) # CPP files for the primary application library under test
+        arg_sets << cpp_files_libraries(@full_dependencies).map(&:to_s) # CPP files for all the libraries we depend on
+        args = arg_sets.flatten(1)
+        return nil unless run_gcc(gcc_binary, *args)
+      end
 
       artifacts << executable
       executable