Skip to content

Separated library sources in cmake for selective. #5136

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Jan 31, 2024
Merged

Separated library sources in cmake for selective. #5136

Changes from 3 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
a90050c
Separated library sources in cmake for selective.
AronRubin Apr 30, 2021
4aef418
Reodered selective process to match CI script
AronRubin Apr 30, 2021
71bad8a
Fixed missing SimpleBLE in library list
AronRubin Apr 30, 2021
0982989
Merge branch 'master' into selective_libraries_with_cmake
P-R-O-C-H-Y Jan 31, 2024
05f6406
fix(cmake): Remove duplicate or non existing sources
P-R-O-C-H-Y Jan 31, 2024
ae71362
fix(cmake): Remove required component
P-R-O-C-H-Y Jan 31, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
182 changes: 111 additions & 71 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,44 +40,106 @@ set(CORE_SRCS
cores/esp32/WString.cpp
)

set(LIBRARY_SRCS
libraries/ArduinoOTA/src/ArduinoOTA.cpp
libraries/AsyncUDP/src/AsyncUDP.cpp
set(ARDUINO_ALL_LIBRARIES
ArduinoOTA
AsyncUDP
BLE
BluetoothSerial
DNSServer
EEPROM
ESPmDNS
FFat
FS
HTTPClient
HTTPUpdate
LITTLEFS
NetBIOS
Preferences
RainMaker
SD_MMC
SD
SimpleBLE
SPIFFS
SPI
Ticker
Update
WebServer
WiFiClientSecure
WiFi
WiFiProv
Wire
)

set(ARDUINO_LIBRARY_ArduinoOTA_SRCS libraries/ArduinoOTA/src/ArduinoOTA.cpp)
set(ARDUINO_LIBRARY_ArduinoOTA_REQUIRES esp_https_ota)

set(ARDUINO_LIBRARY_AsyncUDP_SRCS libraries/AsyncUDP/src/AsyncUDP.cpp)

set(ARDUINO_LIBRARY_BluetoothSerial_SRCS
libraries/BluetoothSerial/src/BluetoothSerial.cpp
libraries/BluetoothSerial/src/BTAddress.cpp
libraries/BluetoothSerial/src/BTAdvertisedDeviceSet.cpp
libraries/BluetoothSerial/src/BTScanResultsSet.cpp
libraries/DNSServer/src/DNSServer.cpp
libraries/EEPROM/src/EEPROM.cpp
libraries/ESPmDNS/src/ESPmDNS.cpp
libraries/FFat/src/FFat.cpp
libraries/BluetoothSerial/src/BTScanResultsSet.cpp)

set(ARDUINO_LIBRARY_DNSServer_SRCS libraries/DNSServer/src/DNSServer.cpp)

set(ARDUINO_LIBRARY_EEPROM_SRCS libraries/EEPROM/src/EEPROM.cpp)

set(ARDUINO_LIBRARY_ESPmDNS_SRCS libraries/ESPmDNS/src/ESPmDNS.cpp)

set(ARDUINO_LIBRARY_FFat_SRCS libraries/FFat/src/FFat.cpp)

set(ARDUINO_LIBRARY_FS_SRCS
libraries/FS/src/FS.cpp
libraries/FS/src/vfs_api.cpp
libraries/HTTPClient/src/HTTPClient.cpp
libraries/HTTPUpdate/src/HTTPUpdate.cpp
libraries/LITTLEFS/src/LITTLEFS.cpp
libraries/NetBIOS/src/NetBIOS.cpp
libraries/Preferences/src/Preferences.cpp
libraries/FS/src/vfs_api.cpp)

set(ARDUINO_LIBRARY_HTTPClient_SRCS libraries/HTTPClient/src/HTTPClient.cpp)

set(ARDUINO_LIBRARY_HTTPUpdate_SRCS libraries/HTTPUpdate/src/HTTPUpdate.cpp)

set(ARDUINO_LIBRARY_LITTLEFS_SRCS libraries/LITTLEFS/src/LITTLEFS.cpp)
set(ARDUINO_LIBRARY_LITTLEFS_REQUIRES esp_littlefs)

set(ARDUINO_LIBRARY_NetBIOS_SRCS libraries/NetBIOS/src/NetBIOS.cpp)

set(ARDUINO_LIBRARY_Preferences_SRCS libraries/Preferences/src/Preferences.cpp)

set(ARDUINO_LIBRARY_RainMaker_SRCS
libraries/RainMaker/src/RMaker.cpp
libraries/RainMaker/src/RMakerNode.cpp
libraries/RainMaker/src/RMakerParam.cpp
libraries/RainMaker/src/RMakerDevice.cpp
libraries/RainMaker/src/RMakerType.cpp
libraries/SD_MMC/src/SD_MMC.cpp
libraries/RainMaker/src/RMakerType.cpp)

set(ARDUINO_LIBRARY_SD_MMC_SRCS libraries/SD_MMC/src/SD_MMC.cpp)

set(ARDUINO_LIBRARY_SD_SRCS
libraries/SD/src/SD.cpp
libraries/SD/src/sd_diskio.cpp
libraries/SD/src/sd_diskio_crc.c
libraries/SimpleBLE/src/SimpleBLE.cpp
libraries/SPIFFS/src/SPIFFS.cpp
libraries/SPI/src/SPI.cpp
libraries/Ticker/src/Ticker.cpp
libraries/SD/src/sd_diskio_crc.c)

set(ARDUINO_LIBRARY_SimpleBLE_SRCS libraries/SimpleBLE/src/SimpleBLE.cpp)

set(ARDUINO_LIBRARY_SPIFFS_SRCS libraries/SPIFFS/src/SPIFFS.cpp)

set(ARDUINO_LIBRARY_SPI_SRCS libraries/SPI/src/SPI.cpp)

set(ARDUINO_LIBRARY_Ticker_SRCS libraries/Ticker/src/Ticker.cpp)

set(ARDUINO_LIBRARY_Update_SRCS
libraries/Update/src/Updater.cpp
libraries/Update/src/HttpsOTAUpdate.cpp
libraries/Update/src/HttpsOTAUpdate.cpp)

set(ARDUINO_LIBRARY_WebServer_SRCS
libraries/WebServer/src/WebServer.cpp
libraries/WebServer/src/Parsing.cpp
libraries/WebServer/src/detail/mimetable.cpp
libraries/WebServer/src/detail/mimetable.cpp)

set(ARDUINO_LIBRARY_WiFiClientSecure_SRCS
libraries/WiFiClientSecure/src/ssl_client.cpp
libraries/WiFiClientSecure/src/WiFiClientSecure.cpp
libraries/WiFiClientSecure/src/WiFiClientSecure.cpp)

set(ARDUINO_LIBRARY_WiFi_SRCS
libraries/WiFi/src/ETH.cpp
libraries/WiFi/src/WiFiAP.cpp
libraries/WiFi/src/WiFiClient.cpp
Expand All @@ -87,12 +149,13 @@ set(LIBRARY_SRCS
libraries/WiFi/src/WiFiScan.cpp
libraries/WiFi/src/WiFiServer.cpp
libraries/WiFi/src/WiFiSTA.cpp
libraries/WiFi/src/WiFiUdp.cpp
libraries/WiFiProv/src/WiFiProv.cpp
libraries/Wire/src/Wire.cpp
)
libraries/WiFi/src/WiFiUdp.cpp)

set(ARDUINO_LIBRARY_WiFiProv_SRCS libraries/WiFiProv/src/WiFiProv.cpp)

set(BLE_SRCS
set(ARDUINO_LIBRARY_Wire_SRCS libraries/Wire/src/Wire.cpp)

set(ARDUINO_LIBRARY_BLE_SRCS
libraries/BLE/src/BLE2902.cpp
libraries/BLE/src/BLE2904.cpp
libraries/BLE/src/BLEAddress.cpp
Expand Down Expand Up @@ -124,51 +187,28 @@ set(BLE_SRCS
libraries/BLE/src/GeneralUtils.cpp
)

set(ARDUINO_LIBRARIES_SRCS)
set(ARDUINO_LIBRARIES_REQUIRES)
set(ARDUINO_LIBRARIES_INCLUDEDIRS)
foreach(libname IN LISTS ARDUINO_ALL_LIBRARIES)
if(NOT CONFIG_ARDUINO_SELECTIVE_COMPILATION OR CONFIG_ARDUINO_SELECTIVE_${libname})
if(ARDUINO_LIBRARY_${libname}_SRCS)
list(APPEND ARDUINO_LIBRARIES_SRCS ${ARDUINO_LIBRARY_${libname}_SRCS})
endif()
if(ARDUINO_LIBRARY_${libname}_REQUIRES)
list(APPEND ARDUINO_LIBRARIES_REQUIRES ${ARDUINO_LIBRARY_${libname}_REQUIRES})
endif()
if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/libraries/${libname}/src)
list(APPEND ARDUINO_LIBRARIES_INCLUDEDIRS libraries/${libname}/src)
endif()
endif()
endforeach()

set(includedirs
variants/${IDF_TARGET}/
cores/esp32/
libraries/ArduinoOTA/src
libraries/AsyncUDP/src
libraries/BLE/src
libraries/BluetoothSerial/src
libraries/DNSServer/src
libraries/EEPROM/src
libraries/ESP32/src
libraries/ESPmDNS/src
libraries/FFat/src
libraries/FS/src
libraries/HTTPClient/src
libraries/HTTPUpdate/src
libraries/LITTLEFS/src
libraries/NetBIOS/src
libraries/Preferences/src
libraries/RainMaker/src
libraries/SD_MMC/src
libraries/SD/src
libraries/SimpleBLE/src
libraries/SPIFFS/src
libraries/SPI/src
libraries/Ticker/src
libraries/Update/src
libraries/WebServer/src
libraries/WiFiClientSecure/src
libraries/WiFi/src
libraries/WiFiProv/src
libraries/Wire/src
)

set(srcs ${CORE_SRCS} ${LIBRARY_SRCS} ${BLE_SRCS})
set(includedirs variants/${IDF_TARGET}/ cores/esp32/ ${ARDUINO_LIBRARIES_INCLUDEDIRS})
set(srcs ${CORE_SRCS} ${ARDUINO_LIBRARIES_SRCS})
set(priv_includes cores/esp32/libb64)
set(requires spi_flash mbedtls mdns esp_adc_cal)
set(priv_requires fatfs nvs_flash app_update spiffs bootloader_support openssl bt arduino_tinyusb main)

if(NOT CONFIG_ARDUINO_SELECTIVE_COMPILATION OR CONFIG_ARDUINO_SELECTIVE_ArduinoOTA)
list(APPEND priv_requires esp_https_ota)
endif()
if(NOT CONFIG_ARDUINO_SELECTIVE_COMPILATION OR CONFIG_ARDUINO_SELECTIVE_LITTLEFS)
list(APPEND priv_requires esp_littlefs)
endif()
set(priv_requires fatfs nvs_flash app_update spiffs bootloader_support openssl bt main ${ARDUINO_LIBRARIES_REQUIRES})
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This looks like a good way to structure the source file dependencies, nice and clean!

The only difficulty is ARDUINO_LIBRARIES_REQUIRES, because in the ESP-IDF build system component requirements can't depend on config. This is actually already a problem in the current CMakeLists.txt file, though - it's not a problem added in this PR.

What happens in the build system is the component CMakeLists.txt files are expanded twice - once to build the dependency tree of component requirements and determine which components are included in the build, and then again to do everything else (add source files, compiler command line arguments, etc.). The first expansion doesn't have the project config available (because the project config depends on which components are included in the build, so it would otherwise be a cyclic dependency.)

As a result, I expect that the test if(NOT CONFIG_ARDUINO_SELECTIVE_COMPILATION OR CONFIG_ARDUINO_SELECTIVE_${libname}) is always true on the first pass, and all of the component requirements (esp_littlefs, esp_https_ota, etc.) will be added to ARDUINO_LIBRARIES_REQUIRES even if these options are disabled.

On the second pass, the config is available so the SRCS files are filtered correctly as expected.

Long term, we know this is not ideal and we have a plan to add "weak" requirements of the variety "component X requires component Y, but only if component Y is available". But this isn't available right now, sorry!

Suggest for now making the requirements list constant (ie remove ARDUINO_LIBRARIES_REQUIRES, place esp_littlefs and esp_https_ota on this line.)

The rest LGTM!

Copy link
Contributor Author

@AronRubin AronRubin Jun 4, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have actually been able to get around that limitation with another project by using

idf_build_get_property(__hack_component_targets __COMPONENT_TARGETS)

And then testing both __hack_component_targets and BUILD_COMPONENTS in the rest of the CMake file. esp_littlefs requires subprojects and esp_https_ota should not be required for projects that do not use OTA (e.g. are BLE only).

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@projectgus it seems to work though. I tested this exact case when fixing arduino to compile as stand-alone component.

@AronRubin could you please rebase to the current CMakeLists.txt?

Copy link
Contributor

@projectgus projectgus Jul 16, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@me-no-dev Oh you're right - sorry. I see now that in current master that if CONFIG_LITTLEFS_PAGE_SIZE and CONFIG_TINYUSB_ENABLED are not set then the relevant component is not added, so that will work fine. (In the first expansion these variables are always empty as there is no config yet, in the second expansion their value will depend on the config.)

Looking at __COMPONENT_TARGETS to check the list at build time should work as well. You can also do the check this way:

if (NOT CMAKE_BUILD_EARLY_EXPANSION)
    idf_build_get_property(components_to_build BUILD_COMPONENTS)
    if(some_component IN_LIST components_to_build)
      message(STATUS "Including optional component some_component")
      list(APPEND priv_requires some_component)
   endif()
endif()

(The only significant differences: checking CMAKE_BUILD_EARLY_EXPANSION means this is skipped on the initial pass before any config is available, and the BUILD_COMPONENTS property is documented so it shouldn't change in an update.)

EDIT: This is wrong, I forgot that priv_requires is ignored after the CMAKE_BUILD_EARLY_EXPANSION phase. To get correct behaviour need to follow this approach: #5401


idf_component_register(INCLUDE_DIRS ${includedirs} PRIV_INCLUDE_DIRS ${priv_includes} SRCS ${srcs} REQUIRES ${requires} PRIV_REQUIRES ${priv_requires})

Expand Down