diff --git a/.classpath b/.classpath index 4d3d2bcc0fb..1ebf54417dd 100644 --- a/.classpath +++ b/.classpath @@ -4,7 +4,6 @@ - @@ -16,10 +15,10 @@ - + - + diff --git a/app/.classpath b/app/.classpath index 17be607aaf0..c4a129f4538 100644 --- a/app/.classpath +++ b/app/.classpath @@ -39,6 +39,8 @@ + + diff --git a/app/build.xml b/app/build.xml index cc38670adc6..411fe0f6494 100644 --- a/app/build.xml +++ b/app/build.xml @@ -80,6 +80,10 @@ includeAntRuntime="false" debug="true" classpathref="class.path" /> + + + + diff --git a/app/lib/commons-io-2.6.jar b/app/lib/commons-io-2.6.jar new file mode 100644 index 00000000000..00556b119d4 Binary files /dev/null and b/app/lib/commons-io-2.6.jar differ diff --git a/app/lib/log4j-api-2.12.0.jar b/app/lib/log4j-api-2.12.0.jar new file mode 100644 index 00000000000..93f770d64a9 Binary files /dev/null and b/app/lib/log4j-api-2.12.0.jar differ diff --git a/app/lib/log4j-core-2.12.0.jar b/app/lib/log4j-core-2.12.0.jar new file mode 100644 index 00000000000..fbab720635d Binary files /dev/null and b/app/lib/log4j-core-2.12.0.jar differ diff --git a/app/src/log4j2.xml b/app/src/log4j2.xml new file mode 100644 index 00000000000..67338ab8a09 --- /dev/null +++ b/app/src/log4j2.xml @@ -0,0 +1,29 @@ + + + + + + + + + + + + + %d{yyyy-MM-dd'T'HH:mm:ss.SSSXXX}{UTC} %p %c{1.} [%t] %m%n + + + + + + + + + + + + + + diff --git a/app/src/processing/app/Base.java b/app/src/processing/app/Base.java index 9b3fa03f8da..623c077ef87 100644 --- a/app/src/processing/app/Base.java +++ b/app/src/processing/app/Base.java @@ -26,9 +26,11 @@ import cc.arduino.Constants; import cc.arduino.UpdatableBoardsLibsFakeURLsHandler; import cc.arduino.UploaderUtils; -import cc.arduino.packages.Uploader; import cc.arduino.contributions.*; -import cc.arduino.contributions.libraries.*; +import cc.arduino.contributions.libraries.ContributedLibrary; +import cc.arduino.contributions.libraries.LibrariesIndexer; +import cc.arduino.contributions.libraries.LibraryInstaller; +import cc.arduino.contributions.libraries.LibraryOfSameTypeComparator; import cc.arduino.contributions.libraries.ui.LibraryManagerUI; import cc.arduino.contributions.packages.ContributedPlatform; import cc.arduino.contributions.packages.ContributionInstaller; @@ -36,20 +38,17 @@ import cc.arduino.contributions.packages.ui.ContributionManagerUI; import cc.arduino.files.DeleteFilesOnShutdown; import cc.arduino.packages.DiscoveryManager; +import cc.arduino.packages.Uploader; import cc.arduino.view.Event; import cc.arduino.view.JMenuUtils; import cc.arduino.view.SplashScreenHelper; - +import com.github.zafarkhaja.semver.Version; import org.apache.commons.compress.utils.IOUtils; import org.apache.commons.lang3.StringUtils; - -import com.github.zafarkhaja.semver.Version; - import processing.app.debug.TargetBoard; import processing.app.debug.TargetPackage; import processing.app.debug.TargetPlatform; import processing.app.helpers.*; -import processing.app.helpers.OSUtils; import processing.app.helpers.filefilters.OnlyDirs; import processing.app.helpers.filefilters.OnlyFilesWithExtension; import processing.app.javax.swing.filechooser.FileNameExtensionFilter; @@ -67,9 +66,9 @@ import java.awt.*; import java.awt.event.*; import java.io.*; -import java.util.*; import java.util.List; import java.util.Timer; +import java.util.*; import java.util.logging.Handler; import java.util.logging.Level; import java.util.logging.Logger; @@ -208,6 +207,8 @@ public Base(String[] args) throws Exception { BaseNoGui.getPlatform().init(); BaseNoGui.initPortableFolder(); + // This configure the logs root folder + System.setProperty("log4j.saveDirectory", BaseNoGui.getSettingsFolder().getAbsolutePath()); // Look for a possible "--preferences-file" parameter and load preferences BaseNoGui.initParameters(args); @@ -286,8 +287,9 @@ public Base(String[] args) throws Exception { pdeKeywords = new PdeKeywords(); pdeKeywords.reload(); - contributionInstaller = new ContributionInstaller(BaseNoGui.getPlatform(), new GPGDetachedSignatureVerifier()); - libraryInstaller = new LibraryInstaller(BaseNoGui.getPlatform()); + final GPGDetachedSignatureVerifier gpgDetachedSignatureVerifier = new GPGDetachedSignatureVerifier(); + contributionInstaller = new ContributionInstaller(BaseNoGui.getPlatform(), gpgDetachedSignatureVerifier); + libraryInstaller = new LibraryInstaller(BaseNoGui.getPlatform(), gpgDetachedSignatureVerifier); parser.parseArgumentsPhase2(); @@ -301,7 +303,7 @@ public Base(String[] args) throws Exception { if (parser.isInstallBoard()) { ContributionsIndexer indexer = new ContributionsIndexer( BaseNoGui.getSettingsFolder(), BaseNoGui.getHardwareFolder(), - BaseNoGui.getPlatform(), new GPGDetachedSignatureVerifier()); + BaseNoGui.getPlatform(), gpgDetachedSignatureVerifier); ProgressListener progressListener = new ConsoleProgressListener(); List downloadedPackageIndexFiles = contributionInstaller.updateIndex(progressListener); diff --git a/arduino-core/.classpath b/arduino-core/.classpath index ad5d49b4bb4..b2f95937e29 100644 --- a/arduino-core/.classpath +++ b/arduino-core/.classpath @@ -8,6 +8,8 @@ + + diff --git a/arduino-core/lib/commons-io-2.6.jar b/arduino-core/lib/commons-io-2.6.jar new file mode 100644 index 00000000000..00556b119d4 Binary files /dev/null and b/arduino-core/lib/commons-io-2.6.jar differ diff --git a/arduino-core/lib/log4j-api-2.12.0.jar b/arduino-core/lib/log4j-api-2.12.0.jar new file mode 100644 index 00000000000..93f770d64a9 Binary files /dev/null and b/arduino-core/lib/log4j-api-2.12.0.jar differ diff --git a/arduino-core/lib/log4j-core-2.12.0.jar b/arduino-core/lib/log4j-core-2.12.0.jar new file mode 100644 index 00000000000..fbab720635d Binary files /dev/null and b/arduino-core/lib/log4j-core-2.12.0.jar differ diff --git a/arduino-core/src/cc/arduino/contributions/DownloadableContributionsDownloader.java b/arduino-core/src/cc/arduino/contributions/DownloadableContributionsDownloader.java index 3157514f876..ba92f89050c 100644 --- a/arduino-core/src/cc/arduino/contributions/DownloadableContributionsDownloader.java +++ b/arduino-core/src/cc/arduino/contributions/DownloadableContributionsDownloader.java @@ -30,20 +30,25 @@ package cc.arduino.contributions; import cc.arduino.utils.FileHash; +import cc.arduino.utils.MultiStepProgress; import cc.arduino.utils.Progress; import cc.arduino.utils.network.FileDownloader; +import org.apache.commons.io.FilenameUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import processing.app.BaseNoGui; +import processing.app.PreferencesData; import java.io.File; import java.net.URL; -import java.nio.file.Files; -import java.nio.file.LinkOption; -import java.nio.file.Path; -import java.nio.file.Paths; +import java.nio.file.*; +import java.util.Collection; import static processing.app.I18n.format; import static processing.app.I18n.tr; public class DownloadableContributionsDownloader { + private static Logger log = LogManager.getLogger(DownloadableContributionsDownloader.class); private final File stagingFolder; @@ -51,11 +56,11 @@ public DownloadableContributionsDownloader(File _stagingFolder) { stagingFolder = _stagingFolder; } - public File download(DownloadableContribution contribution, Progress progress, final String statusText, ProgressListener progressListener) throws Exception { - return download(contribution, progress, statusText, progressListener, false); + public File download(DownloadableContribution contribution, Progress progress, final String statusText, ProgressListener progressListener, boolean allowCache) throws Exception { + return download(contribution, progress, statusText, progressListener, false, allowCache); } - public File download(DownloadableContribution contribution, Progress progress, final String statusText, ProgressListener progressListener, boolean noResume) throws Exception { + public File download(DownloadableContribution contribution, Progress progress, final String statusText, ProgressListener progressListener, boolean noResume, boolean allowCache) throws Exception { URL url = new URL(contribution.getUrl()); Path outputFile = Paths.get(stagingFolder.getAbsolutePath(), contribution.getArchiveFileName()); @@ -70,7 +75,7 @@ public File download(DownloadableContribution contribution, Progress progress, f while (true) { // Need to download or resume downloading? if (!Files.isRegularFile(outputFile, LinkOption.NOFOLLOW_LINKS) || (Files.size(outputFile) < contribution.getSize())) { - download(url, outputFile.toFile(), progress, statusText, progressListener, noResume); + download(url, outputFile.toFile(), progress, statusText, progressListener, noResume, allowCache); downloaded = true; } @@ -116,12 +121,12 @@ private boolean hasChecksum(DownloadableContribution contribution) { return algo != null && !algo.isEmpty(); } - public void download(URL url, File tmpFile, Progress progress, String statusText, ProgressListener progressListener) throws Exception { - download(url, tmpFile, progress, statusText, progressListener, false); + public void download(URL url, File tmpFile, Progress progress, String statusText, ProgressListener progressListener, boolean allowCache) throws Exception { + download(url, tmpFile, progress, statusText, progressListener, false, allowCache); } - public void download(URL url, File tmpFile, Progress progress, String statusText, ProgressListener progressListener, boolean noResume) throws Exception { - FileDownloader downloader = new FileDownloader(url, tmpFile); + public void download(URL url, File tmpFile, Progress progress, String statusText, ProgressListener progressListener, boolean noResume, boolean allowCache) throws Exception { + final FileDownloader downloader = new FileDownloader(url, tmpFile, allowCache); downloader.addObserver((o, arg) -> { FileDownloader me = (FileDownloader) o; String msg = ""; @@ -140,4 +145,94 @@ public void download(URL url, File tmpFile, Progress progress, String statusText } } + public void downloadIndexAndSignature(MultiStepProgress progress, URL packageIndexUrl, ProgressListener progressListener, SignatureVerifier signatureVerifier) throws Exception { + + // Extract the file name from the url + final String indexFileName = FilenameUtils.getName(packageIndexUrl.getPath()); + final File packageIndex = BaseNoGui.indexer.getIndexFile(indexFileName); + + final String statusText = tr("Downloading platforms index..."); + + // Create temp files + final File packageIndexTemp = File.createTempFile(indexFileName, ".tmp"); + try { + // Download package index + download(packageIndexUrl, packageIndexTemp, progress, statusText, progressListener, true, true); + final URL signatureUrl = new URL(packageIndexUrl.toString() + ".sig"); + + if (verifyDomain(packageIndexUrl)) { + if (checkSignature(progress, signatureUrl, progressListener, signatureVerifier, statusText, packageIndexTemp)) { + Files.move(packageIndexTemp.toPath(), packageIndex.toPath(), StandardCopyOption.REPLACE_EXISTING); + } else { + log.info("The cached files have been removed. {} {}", packageIndexUrl, signatureUrl); + FileDownloader.invalidateFiles(packageIndexUrl, signatureUrl); + } + } else { + // Move the package index to the destination when the signature is not necessary + Files.move(packageIndexTemp.toPath(), packageIndex.toPath(), StandardCopyOption.REPLACE_EXISTING); + log.info("The domain is not selected to verify the signature. will be copied into this path {}, packageIndex url: {}", packageIndex, packageIndexUrl); + } + } catch (Exception e) { + log.error("Cannot download the package index from {} the package will be discard", packageIndexUrl, e); + throw e; + } finally { + // Delete useless temp file + Files.deleteIfExists(packageIndexTemp.toPath()); + } + } + + public boolean verifyDomain(URL url) { + final Collection domain = PreferencesData. + getCollection("http.signature_verify_domains"); + if (domain.size() == 0) { + // Default domain + domain.add("downloads.arduino.cc"); + } + if (domain.contains(url.getHost())) { + return true; + } else { + log.info("The domain is not selected to verify the signature. domain list: {}, url: {}", domain, url); + return false; + } + } + + public boolean checkSignature(MultiStepProgress progress, URL signatureUrl, ProgressListener progressListener, SignatureVerifier signatureVerifier, String statusText, File fileToVerify) throws Exception { + + final boolean allowInsecurePackages = + PreferencesData.getBoolean("allow_insecure_packages", false); + if (allowInsecurePackages) { + log.info("Allow insecure packages is true the signature will be skip and return always verified"); + return true; + } + + // Signature file name + final String signatureFileName = FilenameUtils.getName(signatureUrl.getPath()); + final File packageIndexSignature = BaseNoGui.indexer.getIndexFile(signatureFileName); + final File packageIndexSignatureTemp = File.createTempFile(signatureFileName, ".tmp"); + + + try { + // Download signature + download(signatureUrl, packageIndexSignatureTemp, progress, statusText, progressListener, true); + + // Verify the signature before move the files + final boolean signatureVerified = signatureVerifier.isSigned(fileToVerify, packageIndexSignatureTemp); + if (signatureVerified) { + log.info("Signature verified. url={}, signature url={}, file to verify={}, signature file={}", signatureUrl, signatureUrl, fileToVerify, packageIndexSignatureTemp); + // Move if the signature is ok + Files.move(packageIndexSignatureTemp.toPath(), packageIndexSignature.toPath(), StandardCopyOption.REPLACE_EXISTING); + } else { + log.error("{} file signature verification failed. File ignored.", signatureUrl); + System.err.println(format(tr("{0} file signature verification failed. File ignored."), signatureUrl.toString())); + } + return signatureVerified; + } catch (Exception e) { + log.error("Cannot download the signature from {} the package will be discard", signatureUrl, e); + throw e; + } finally { + Files.deleteIfExists(packageIndexSignatureTemp.toPath()); + } + + } + } diff --git a/arduino-core/src/cc/arduino/contributions/GZippedJsonDownloader.java b/arduino-core/src/cc/arduino/contributions/GZippedJsonDownloader.java index 6b6f3812327..8a717dcf26c 100644 --- a/arduino-core/src/cc/arduino/contributions/GZippedJsonDownloader.java +++ b/arduino-core/src/cc/arduino/contributions/GZippedJsonDownloader.java @@ -29,13 +29,16 @@ package cc.arduino.contributions; +import cc.arduino.Constants; import cc.arduino.utils.Progress; import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; import org.apache.commons.compress.compressors.gzip.GzipUtils; import org.apache.commons.compress.utils.IOUtils; +import org.apache.commons.io.FilenameUtils; import java.io.*; import java.net.URL; +import java.nio.file.Files; public class GZippedJsonDownloader { @@ -49,18 +52,22 @@ public GZippedJsonDownloader(DownloadableContributionsDownloader downloader, URL this.gzippedUrl = gzippedUrl; } - public void download(File tmpFile, Progress progress, String statusText, ProgressListener progressListener) throws Exception { + public void download(File tmpFile, Progress progress, String statusText, ProgressListener progressListener, boolean allowCache) throws Exception { + File gzipTmpFile = null; try { - File gzipTmpFile = new File(tmpFile.getParentFile(), GzipUtils.getCompressedFilename(tmpFile.getName())); + String tmpFileName = FilenameUtils.getName(new URL(Constants.LIBRARY_INDEX_URL_GZ).getPath()); + gzipTmpFile = File.createTempFile(tmpFileName, GzipUtils.getCompressedFilename(tmpFile.getName())); // remove eventual leftovers from previous downloads - if (gzipTmpFile.exists()) { - gzipTmpFile.delete(); - } - new JsonDownloader(downloader, gzippedUrl).download(gzipTmpFile, progress, statusText, progressListener); + Files.deleteIfExists(gzipTmpFile.toPath()); + + new JsonDownloader(downloader, gzippedUrl).download(gzipTmpFile, progress, statusText, progressListener, allowCache); decompress(gzipTmpFile, tmpFile); - gzipTmpFile.delete(); } catch (Exception e) { - new JsonDownloader(downloader, url).download(tmpFile, progress, statusText, progressListener); + new JsonDownloader(downloader, url).download(tmpFile, progress, statusText, progressListener, allowCache); + } finally { + if (gzipTmpFile != null) { + Files.deleteIfExists(gzipTmpFile.toPath()); + } } } diff --git a/arduino-core/src/cc/arduino/contributions/JsonDownloader.java b/arduino-core/src/cc/arduino/contributions/JsonDownloader.java index 88f9e7783f1..5b932d08064 100644 --- a/arduino-core/src/cc/arduino/contributions/JsonDownloader.java +++ b/arduino-core/src/cc/arduino/contributions/JsonDownloader.java @@ -44,9 +44,9 @@ public JsonDownloader(DownloadableContributionsDownloader downloader, URL url) { this.url = url; } - public void download(File tmpFile, Progress progress, String statusText, ProgressListener progressListener) throws Exception { + public void download(File tmpFile, Progress progress, String statusText, ProgressListener progressListener, boolean allowCache) throws Exception { try { - downloader.download(url, tmpFile, progress, statusText, progressListener); + downloader.download(url, tmpFile, progress, statusText, progressListener, allowCache); } catch (InterruptedException e) { // Download interrupted... just exit } diff --git a/arduino-core/src/cc/arduino/contributions/SignatureVerifier.java b/arduino-core/src/cc/arduino/contributions/SignatureVerifier.java index 6e2a80626b1..a4ea7a7ba53 100644 --- a/arduino-core/src/cc/arduino/contributions/SignatureVerifier.java +++ b/arduino-core/src/cc/arduino/contributions/SignatureVerifier.java @@ -50,6 +50,15 @@ public boolean isSigned(File indexFile) { } } + public boolean isSigned(File indexFile, File signature) { + try { + return verify(indexFile, signature, new File(BaseNoGui.getContentFile("lib"), "public.gpg.key")); + } catch (Exception e) { + BaseNoGui.showWarning(e.getMessage(), e.getMessage(), e); + return false; + } + } + protected abstract boolean verify(File signedFile, File signature, File publicKey) throws IOException; } diff --git a/arduino-core/src/cc/arduino/contributions/libraries/LibraryInstaller.java b/arduino-core/src/cc/arduino/contributions/libraries/LibraryInstaller.java index de91c049090..3f00f909b0d 100644 --- a/arduino-core/src/cc/arduino/contributions/libraries/LibraryInstaller.java +++ b/arduino-core/src/cc/arduino/contributions/libraries/LibraryInstaller.java @@ -31,10 +31,15 @@ import cc.arduino.Constants; import cc.arduino.contributions.DownloadableContributionsDownloader; +import cc.arduino.contributions.GPGDetachedSignatureVerifier; import cc.arduino.contributions.GZippedJsonDownloader; import cc.arduino.contributions.ProgressListener; import cc.arduino.utils.ArchiveExtractor; import cc.arduino.utils.MultiStepProgress; +import cc.arduino.utils.network.FileDownloader; +import org.apache.commons.io.FilenameUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import processing.app.BaseNoGui; import processing.app.I18n; import processing.app.Platform; @@ -43,6 +48,8 @@ import java.io.File; import java.io.IOException; import java.net.URL; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -50,11 +57,14 @@ import static processing.app.I18n.tr; public class LibraryInstaller { + private static Logger log = LogManager.getLogger(LibraryInstaller.class); private final Platform platform; + private final GPGDetachedSignatureVerifier signatureVerifier; - public LibraryInstaller(Platform platform) { + public LibraryInstaller(Platform platform, GPGDetachedSignatureVerifier signatureVerifier) { this.platform = platform; + this.signatureVerifier = signatureVerifier; } public synchronized void updateIndex(ProgressListener progressListener) throws Exception { @@ -63,29 +73,42 @@ public synchronized void updateIndex(ProgressListener progressListener) throws E DownloadableContributionsDownloader downloader = new DownloadableContributionsDownloader(BaseNoGui.librariesIndexer.getStagingFolder()); // Step 1: Download index File outputFile = BaseNoGui.librariesIndexer.getIndexFile(); - File tmpFile = new File(outputFile.getAbsolutePath() + ".tmp"); + // Create temp files + String signatureFileName = FilenameUtils.getName(new URL(Constants.LIBRARY_INDEX_URL).getPath()); + File libraryIndexTemp = File.createTempFile(signatureFileName, ".tmp"); + final URL libraryURL = new URL(Constants.LIBRARY_INDEX_URL); + final URL libraryGzURL = new URL(Constants.LIBRARY_INDEX_URL_GZ); + final String statusText = tr("Downloading libraries index..."); try { - GZippedJsonDownloader gZippedJsonDownloader = new GZippedJsonDownloader(downloader, new URL(Constants.LIBRARY_INDEX_URL), new URL(Constants.LIBRARY_INDEX_URL_GZ)); - gZippedJsonDownloader.download(tmpFile, progress, tr("Downloading libraries index..."), progressListener); + GZippedJsonDownloader gZippedJsonDownloader = new GZippedJsonDownloader(downloader, libraryURL, libraryGzURL); + gZippedJsonDownloader.download(libraryIndexTemp, progress, statusText, progressListener, true); } catch (InterruptedException e) { // Download interrupted... just exit return; } progress.stepDone(); - // TODO: Check downloaded index - - // Replace old index with the updated one - if (outputFile.exists()) - outputFile.delete(); - if (!tmpFile.renameTo(outputFile)) - throw new Exception(tr("An error occurred while updating libraries index!")); + URL signatureUrl = new URL(libraryURL.toString() + ".sig"); + if (downloader.verifyDomain(signatureUrl)) { + if (downloader.checkSignature(progress, signatureUrl, progressListener, signatureVerifier, statusText, libraryIndexTemp)) { + // Replace old index with the updated one + if (libraryIndexTemp.length() > 0) { + Files.move(libraryIndexTemp.toPath(), outputFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + } + } else { + FileDownloader.invalidateFiles(libraryGzURL, libraryURL, signatureUrl); + log.error("Fail to verify the signature of {} the cached files have been removed", libraryURL); + } + } else { + log.info("The domain is not selected to verify the signature. library index: {}", signatureUrl); + } // Step 2: Parse index BaseNoGui.librariesIndexer.parseIndex(); // Step 3: Rescan index rescanLibraryIndex(progress, progressListener); + } public void install(ContributedLibrary lib, ProgressListener progressListener) throws Exception { @@ -129,7 +152,7 @@ private void performInstall(ContributedLibrary lib, ProgressListener progressLis // Step 1: Download library try { - downloader.download(lib, progress, I18n.format(tr("Downloading library: {0}"), lib.getName()), progressListener); + downloader.download(lib, progress, I18n.format(tr("Downloading library: {0}"), lib.getName()), progressListener, false); } catch (InterruptedException e) { // Download interrupted... just exit return; diff --git a/arduino-core/src/cc/arduino/contributions/packages/ContributionInstaller.java b/arduino-core/src/cc/arduino/contributions/packages/ContributionInstaller.java index c75d1352a8c..ddcfeea5726 100644 --- a/arduino-core/src/cc/arduino/contributions/packages/ContributionInstaller.java +++ b/arduino-core/src/cc/arduino/contributions/packages/ContributionInstaller.java @@ -41,6 +41,9 @@ import org.apache.commons.exec.DefaultExecutor; import org.apache.commons.exec.Executor; import org.apache.commons.exec.PumpStreamHandler; +import org.apache.commons.io.FilenameUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import processing.app.BaseNoGui; import processing.app.I18n; import processing.app.Platform; @@ -62,6 +65,7 @@ import static processing.app.I18n.tr; public class ContributionInstaller { + private static Logger log = LogManager.getLogger(ContributionInstaller.class); private final Platform platform; private final SignatureVerifier signatureVerifier; @@ -98,7 +102,7 @@ public synchronized List install(ContributedPlatform contributedPlatform // Download all try { // Download platform - downloader.download(contributedPlatform, progress, tr("Downloading boards definitions."), progressListener); + downloader.download(contributedPlatform, progress, tr("Downloading boards definitions."), progressListener, false); progress.stepDone(); // Download tools @@ -106,7 +110,7 @@ public synchronized List install(ContributedPlatform contributedPlatform for (ContributedTool tool : tools) { String msg = format(tr("Downloading tools ({0}/{1})."), i, tools.size()); i++; - downloader.download(tool.getDownloadableContribution(platform), progress, msg, progressListener); + downloader.download(tool.getDownloadableContribution(platform), progress, msg, progressListener, false); progress.stepDone(); } } catch (InterruptedException e) { @@ -122,10 +126,10 @@ public synchronized List install(ContributedPlatform contributedPlatform // all the temporary folders and abort installation. List> resolvedToolReferences = contributedPlatform - .getResolvedToolReferences().entrySet().stream() - .filter((entry) -> !entry.getValue().isInstalled() - || entry.getValue().isBuiltIn()) - .collect(Collectors.toList()); + .getResolvedToolReferences().entrySet().stream() + .filter((entry) -> !entry.getValue().isInstalled() + || entry.getValue().isBuiltIn()) + .collect(Collectors.toList()); int i = 1; for (Map.Entry entry : resolvedToolReferences) { @@ -268,6 +272,8 @@ public synchronized List remove(ContributedPlatform contributedPlatform) Files.delete(destFolder.getParentFile().toPath()); } catch (Exception e) { // ignore + log.info("The directory is not empty there is another version installed. directory {}", + destFolder.getParentFile().toPath(), e); } } @@ -278,74 +284,48 @@ public synchronized List remove(ContributedPlatform contributedPlatform) return errors; } - public synchronized List updateIndex(ProgressListener progressListener) throws Exception { + public synchronized List updateIndex(ProgressListener progressListener) { MultiStepProgress progress = new MultiStepProgress(1); - List downloadedPackageIndexFilesAccumulator = new LinkedList<>(); - downloadIndexAndSignature(progress, downloadedPackageIndexFilesAccumulator, Constants.PACKAGE_INDEX_URL, progressListener); + final DownloadableContributionsDownloader downloader = new DownloadableContributionsDownloader(BaseNoGui.indexer.getStagingFolder()); - Set packageIndexURLs = new HashSet<>(); - String additionalURLs = PreferencesData.get(Constants.PREF_BOARDS_MANAGER_ADDITIONAL_URLS, ""); - if (!"".equals(additionalURLs)) { - packageIndexURLs.addAll(Arrays.asList(additionalURLs.split(","))); - } + final Set packageIndexURLs = new HashSet<>( + PreferencesData.getCollection(Constants.PREF_BOARDS_MANAGER_ADDITIONAL_URLS) + ); + packageIndexURLs.add(Constants.PACKAGE_INDEX_URL); + List downloadedPackageIndexFilesAccumulator = new LinkedList<>(); - for (String packageIndexURL : packageIndexURLs) { + for (String packageIndexURLString : packageIndexURLs) { try { - downloadIndexAndSignature(progress, downloadedPackageIndexFilesAccumulator, packageIndexURL, progressListener); + // Extract the file name from the URL + final URL packageIndexURL = new URL(packageIndexURLString); + String indexFileName = FilenameUtils.getName(packageIndexURL.getPath()); + downloadedPackageIndexFilesAccumulator.add(BaseNoGui.indexer.getIndexFile(indexFileName).getName()); + + log.info("Start download and signature check of={}", packageIndexURLs); + downloader.downloadIndexAndSignature(progress, packageIndexURL, progressListener, signatureVerifier); } catch (Exception e) { + log.error(e.getMessage(), e); System.err.println(e.getMessage()); } } progress.stepDone(); - + log.info("Downloaded package index URL={}", packageIndexURLs); return downloadedPackageIndexFilesAccumulator; } - private void downloadIndexAndSignature(MultiStepProgress progress, List downloadedPackagedIndexFilesAccumulator, String packageIndexUrl, ProgressListener progressListener) throws Exception { - File packageIndex = download(progress, packageIndexUrl, progressListener); - downloadedPackagedIndexFilesAccumulator.add(packageIndex.getName()); - try { - File packageIndexSignature = download(progress, packageIndexUrl + ".sig", progressListener); - boolean signatureVerified = signatureVerifier.isSigned(packageIndex); - if (signatureVerified) { - downloadedPackagedIndexFilesAccumulator.add(packageIndexSignature.getName()); - } else { - downloadedPackagedIndexFilesAccumulator.remove(packageIndex.getName()); - Files.delete(packageIndex.toPath()); - Files.delete(packageIndexSignature.toPath()); - System.err.println(I18n.format(tr("{0} file signature verification failed. File ignored."), packageIndexUrl)); - } - } catch (Exception e) { - //ignore errors - } - } - - private File download(MultiStepProgress progress, String packageIndexUrl, ProgressListener progressListener) throws Exception { - String statusText = tr("Downloading platforms index..."); - URL url = new URL(packageIndexUrl); - String[] urlPathParts = url.getFile().split("/"); - File outputFile = BaseNoGui.indexer.getIndexFile(urlPathParts[urlPathParts.length - 1]); - File tmpFile = new File(outputFile.getAbsolutePath() + ".tmp"); - DownloadableContributionsDownloader downloader = new DownloadableContributionsDownloader(BaseNoGui.indexer.getStagingFolder()); - boolean noResume = true; - downloader.download(url, tmpFile, progress, statusText, progressListener, noResume); - - Files.deleteIfExists(outputFile.toPath()); - Files.move(tmpFile.toPath(), outputFile.toPath()); - - return outputFile; - } - public synchronized void deleteUnknownFiles(List downloadedPackageIndexFiles) throws IOException { File preferencesFolder = BaseNoGui.indexer.getIndexFile(".").getParentFile(); File[] additionalPackageIndexFiles = preferencesFolder.listFiles(new PackageIndexFilenameFilter(Constants.DEFAULT_INDEX_FILE_NAME)); if (additionalPackageIndexFiles == null) { return; } + log.info("Check unknown files. Additional package index folder files={}, Additional package index url downloaded={}", downloadedPackageIndexFiles, additionalPackageIndexFiles); + for (File additionalPackageIndexFile : additionalPackageIndexFiles) { if (!downloadedPackageIndexFiles.contains(additionalPackageIndexFile.getName())) { + log.info("Delete this unknown file={} because not included in this list={}", additionalPackageIndexFile, additionalPackageIndexFiles); Files.delete(additionalPackageIndexFile.toPath()); } } diff --git a/arduino-core/src/cc/arduino/utils/network/CacheControl.java b/arduino-core/src/cc/arduino/utils/network/CacheControl.java new file mode 100644 index 00000000000..a34fde7be3b --- /dev/null +++ b/arduino-core/src/cc/arduino/utils/network/CacheControl.java @@ -0,0 +1,119 @@ +/* + * This file is part of Arduino. + * + * Copyright 2019 Arduino LLC (http://www.arduino.cc/) + * + * Arduino is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As a special exception, you may use this file as part of a free software + * library without restriction. Specifically, if other files instantiate + * templates or use macros or inline functions from this file, or you compile + * this file and link it with other files to produce an executable, this + * file does not by itself cause the resulting executable to be covered by + * the GNU General Public License. This exception does not however + * invalidate any other reasons why the executable file might be covered by + * the GNU General Public License. + */ + + +package cc.arduino.utils.network; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class CacheControl { + + + // see org.apache.abdera.protocol.util.CacheControlUtil + private static final Pattern PATTERN + = Pattern.compile("\\s*([\\w\\-]+)\\s*(=)?\\s*(\\-?\\d+|\\\"([^\"\\\\]*(\\\\.[^\"\\\\]*)*)+\\\")?\\s*"); + + private int maxAge = -1; + + private boolean isMustRevalidate = false; + + private boolean isNoCache = false; + + private boolean isNoStore = false; + + + public static CacheControl valueOf(String value) { + CacheControl cc = new CacheControl(); + + if (value != null) { + Matcher matcher = PATTERN.matcher(value); + while (matcher.find()) { + switch (matcher.group(1).toLowerCase()) { + case "max-age": + cc.setMaxAge(Integer.parseInt(matcher.group(3))); + break; + case "must-revalidate": + cc.setMustRevalidate(true); + break; + case "no-cache": + cc.setNoCache(true); + break; + case "no-store": + cc.setNoStore(true); + break; + default: //ignore + } + } + } + return cc; + } + + public void setMaxAge(int maxAge) { + this.maxAge = maxAge; + } + + public int getMaxAge() { + return maxAge; + } + + public boolean isMustRevalidate() { + return isMustRevalidate; + } + + public void setMustRevalidate(boolean mustRevalidate) { + isMustRevalidate = mustRevalidate; + } + + public boolean isNoCache() { + return isNoCache; + } + + public void setNoCache(boolean noCache) { + isNoCache = noCache; + } + + public boolean isNoStore() { + return isNoStore; + } + + public void setNoStore(boolean noStore) { + isNoStore = noStore; + } + + @Override + public String toString() { + return "CacheControl{" + + "maxAge=" + maxAge + + ", isMustRevalidate=" + isMustRevalidate + + ", isNoCache=" + isNoCache + + ", isNoStore=" + isNoStore + + '}'; + } +} diff --git a/arduino-core/src/cc/arduino/utils/network/FileDownloader.java b/arduino-core/src/cc/arduino/utils/network/FileDownloader.java index 8e25cc9227c..95138d8ff66 100644 --- a/arduino-core/src/cc/arduino/utils/network/FileDownloader.java +++ b/arduino-core/src/cc/arduino/utils/network/FileDownloader.java @@ -29,26 +29,28 @@ package cc.arduino.utils.network; -import cc.arduino.net.CustomProxySelector; -import org.apache.commons.codec.binary.Base64; import org.apache.commons.compress.utils.IOUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import processing.app.helpers.FileUtils; -import processing.app.BaseNoGui; -import processing.app.PreferencesData; - +import javax.script.ScriptException; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.RandomAccessFile; import java.net.HttpURLConnection; -import java.net.Proxy; import java.net.SocketTimeoutException; +import java.net.URISyntaxException; import java.net.URL; import java.nio.file.Files; import java.nio.file.Paths; +import java.util.Arrays; import java.util.Observable; +import java.util.Optional; public class FileDownloader extends Observable { + private static Logger log = LogManager.getLogger(FileDownloader.class); public enum Status { CONNECTING, // @@ -66,17 +68,15 @@ public enum Status { private final URL downloadUrl; private final File outputFile; - private InputStream stream = null; + private final boolean allowCache; private Exception error; - private String userAgent; - - public FileDownloader(URL url, File file) { - downloadUrl = url; - outputFile = file; - downloaded = 0; - initialSize = 0; - userAgent = "ArduinoIDE/" + BaseNoGui.VERSION_NAME + " Java/" - + System.getProperty("java.version"); + + public FileDownloader(URL url, File file, boolean allowCache) { + this.downloadUrl = url; + this.outputFile = file; + this.allowCache = allowCache; + this.downloaded = 0; + this.initialSize = 0; } public long getInitialSize() { @@ -121,9 +121,6 @@ public void setStatus(Status status) { notifyObservers(); } - public void download() throws InterruptedException { - download(false); - } public void download(boolean noResume) throws InterruptedException { if ("file".equals(downloadUrl.getProtocol())) { @@ -143,68 +140,119 @@ private void saveLocalFile() { } } + public static void invalidateFiles(URL... filesUrl) { + // For each file delete the file cached if exist + Arrays.stream(filesUrl).forEach(url -> { + try { + FileDownloaderCache.getFileCached(url).ifPresent(fileCached -> { + try { + log.info("Invalidate this file {} that comes from {}", fileCached.getLocalPath(), fileCached.getRemoteURL()); + fileCached.invalidateCache(); + } catch (Exception e) { + log.warn("Fail to invalidate cache", e); + } + }); + } catch (URISyntaxException | NoSuchMethodException | ScriptException | IOException e) { + log.warn("Fail to get the file cached during the file invalidation", e); + } + }); + + } + private void downloadFile(boolean noResume) throws InterruptedException { - RandomAccessFile file = null; try { - // Open file and seek to the end of it - file = new RandomAccessFile(outputFile, "rw"); - initialSize = file.length(); - - if (noResume && initialSize > 0) { - // delete file and restart downloading - Files.delete(outputFile.toPath()); - initialSize = 0; - } + setStatus(Status.CONNECTING); - file.seek(initialSize); + final Optional fileCachedOpt = FileDownloaderCache.getFileCached(downloadUrl, allowCache); + if (fileCachedOpt.isPresent()) { + final FileDownloaderCache.FileCached fileCached = fileCachedOpt.get(); - setStatus(Status.CONNECTING); + final Optional fileFromCache = getFileCached(fileCached); + if (fileCached.isNotChange() && fileFromCache.isPresent()) { + // Copy the cached file in the destination file + FileUtils.copyFile(fileFromCache.get(), outputFile); + } else { + openConnectionAndFillTheFile(noResume); - Proxy proxy = new CustomProxySelector(PreferencesData.getMap()).getProxyFor(downloadUrl.toURI()); - if ("true".equals(System.getProperty("DEBUG"))) { - System.err.println("Using proxy " + proxy); + fileCached.updateCacheFile(outputFile); + } + } else { + openConnectionAndFillTheFile(noResume); } + setStatus(Status.COMPLETE); - HttpURLConnection connection = (HttpURLConnection) downloadUrl.openConnection(proxy); - connection.setRequestProperty("User-agent", userAgent); - if (downloadUrl.getUserInfo() != null) { - String auth = "Basic " + new String(new Base64().encode(downloadUrl.getUserInfo().getBytes())); - connection.setRequestProperty("Authorization", auth); - } + } catch (InterruptedException e) { + setStatus(Status.CANCELLED); + // lets InterruptedException go up to the caller + throw e; - connection.setRequestProperty("Range", "bytes=" + initialSize + "-"); - connection.setConnectTimeout(5000); - setDownloaded(0); + } catch (SocketTimeoutException e) { + setStatus(Status.CONNECTION_TIMEOUT_ERROR); + setError(e); + log.error("The request went in socket timeout", e); - // Connect - connection.connect(); - int resp = connection.getResponseCode(); + } catch (Exception e) { + setStatus(Status.ERROR); + setError(e); + log.error("The request stop", e); + } - if (resp == HttpURLConnection.HTTP_MOVED_PERM || resp == HttpURLConnection.HTTP_MOVED_TEMP) { - URL newUrl = new URL(connection.getHeaderField("Location")); + } - proxy = new CustomProxySelector(PreferencesData.getMap()).getProxyFor(newUrl.toURI()); + private Optional getFileCached(FileDownloaderCache.FileCached fileCached) { - // open the new connnection again - connection = (HttpURLConnection) newUrl.openConnection(proxy); - connection.setRequestProperty("User-agent", userAgent); - if (downloadUrl.getUserInfo() != null) { - String auth = "Basic " + new String(new Base64().encode(downloadUrl.getUserInfo().getBytes())); - connection.setRequestProperty("Authorization", auth); - } + try { + final Optional fileFromCache = + fileCached.getFileFromCache(); + if (fileFromCache.isPresent()) { + log.info("No need to download using cached file: {}", fileCached); + return fileFromCache; + } else { + log.info( + "The file in the cache is not in the path or the md5 validation failed: path={}, file exist={}, md5 validation={}", + fileCached.getLocalPath(), fileCached.exists(), fileCached.md5Check()); + } + } catch (Exception e) { + log.warn( + "Cannot get the file from the cache, will be downloaded a new one ", e); + } + log.info("The file is change {}", fileCached); + return Optional.empty(); + } - connection.setRequestProperty("Range", "bytes=" + initialSize + "-"); - connection.setConnectTimeout(5000); + private void openConnectionAndFillTheFile(boolean noResume) throws Exception { + initialSize = outputFile.length(); + if (noResume && initialSize > 0) { + // delete file and restart downloading + Files.deleteIfExists(outputFile.toPath()); + initialSize = 0; + } - connection.connect(); - resp = connection.getResponseCode(); - } + final HttpURLConnection connection = new HttpConnectionManager(downloadUrl) + .makeConnection((c) -> setDownloaded(0)); + final int resp = connection.getResponseCode(); - if (resp < 200 || resp >= 300) { - throw new IOException("Received invalid http status code from server: " + resp); - } + if (resp < 200 || resp >= 300) { + Files.deleteIfExists(outputFile.toPath()); + throw new IOException("Received invalid http status code from server: " + resp); + } + + RandomAccessFile randomAccessOutputFile = null; + try { + // Open file and seek to the end of it + randomAccessOutputFile = new RandomAccessFile(outputFile, "rw"); + randomAccessOutputFile.seek(initialSize); + readStreamCopyTo(randomAccessOutputFile, connection); + } finally { + IOUtils.closeQuietly(randomAccessOutputFile); + } + + } + private void readStreamCopyTo(RandomAccessFile randomAccessOutputFile, HttpURLConnection connection) throws Exception { + InputStream stream = null; + try { // Check for valid content length. long len = connection.getContentLength(); if (len >= 0) { @@ -212,20 +260,19 @@ private void downloadFile(boolean noResume) throws InterruptedException { } setStatus(Status.DOWNLOADING); - synchronized (this) { - stream = connection.getInputStream(); - } - byte buffer[] = new byte[10240]; + stream = connection.getInputStream(); + + byte[] buffer = new byte[10240]; while (status == Status.DOWNLOADING) { int read = stream.read(buffer); if (read == -1) break; - file.write(buffer, 0, read); + randomAccessOutputFile.write(buffer, 0, read); setDownloaded(getDownloaded() + read); if (Thread.interrupted()) { - file.close(); + randomAccessOutputFile.close(); throw new InterruptedException(); } } @@ -234,26 +281,8 @@ private void downloadFile(boolean noResume) throws InterruptedException { if (getDownloaded() < getDownloadSize()) throw new Exception("Incomplete download"); } - setStatus(Status.COMPLETE); - } catch (InterruptedException e) { - setStatus(Status.CANCELLED); - // lets InterruptedException go up to the caller - throw e; - - } catch (SocketTimeoutException e) { - setStatus(Status.CONNECTION_TIMEOUT_ERROR); - setError(e); - - } catch (Exception e) { - setStatus(Status.ERROR); - setError(e); - } finally { - IOUtils.closeQuietly(file); - - synchronized (this) { - IOUtils.closeQuietly(stream); - } + IOUtils.closeQuietly(stream); } } diff --git a/arduino-core/src/cc/arduino/utils/network/FileDownloaderCache.java b/arduino-core/src/cc/arduino/utils/network/FileDownloaderCache.java new file mode 100644 index 00000000000..4fe38689f2d --- /dev/null +++ b/arduino-core/src/cc/arduino/utils/network/FileDownloaderCache.java @@ -0,0 +1,425 @@ +/* + * This file is part of Arduino. + * + * Copyright 2019 Arduino LLC (http://www.arduino.cc/) + * + * Arduino is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As a special exception, you may use this file as part of a free software + * library without restriction. Specifically, if other files instantiate + * templates or use macros or inline functions from this file, or you compile + * this file and link it with other files to produce an executable, this + * file does not by itself cause the resulting executable to be covered by + * the GNU General Public License. This exception does not however + * invalidate any other reasons why the executable file might be covered by + * the GNU General Public License. + */ + + +package cc.arduino.utils.network; + +import cc.arduino.utils.FileHash; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import processing.app.BaseNoGui; +import processing.app.PreferencesData; +import processing.app.helpers.FileUtils; + +import javax.script.ScriptException; +import java.io.File; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.ProtocolException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.NoSuchAlgorithmException; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.*; +import java.util.function.BiFunction; +import java.util.function.Function; +import java.util.stream.Collectors; + +public class FileDownloaderCache { + private final static String CACHE_ENABLE_PREFERENCE_KEY = "cache.enable"; + private final static Logger log = LogManager + .getLogger(FileDownloaderCache.class); + private final static Map cachedFiles = Collections + .synchronizedMap(new HashMap<>()); + private final static String cacheFolder; + private static boolean enableCache; + + static { + enableCache = Boolean.valueOf(PreferencesData.get(CACHE_ENABLE_PREFERENCE_KEY, "true")); + if (!enableCache) { + log.info("The cache is disable cache.enable=false"); + } + PreferencesData.set(CACHE_ENABLE_PREFERENCE_KEY, Boolean.toString(enableCache)); + + final File settingsFolder; + settingsFolder = BaseNoGui.getSettingsFolder(); + if (settingsFolder != null) { + cacheFolder = Paths.get(settingsFolder.getPath(), "cache") + .toString(); + } else { + enableCache = false; + cacheFolder = null; + log.error("The cache will disable because the setting folder is null, cannot generate the cache path"); + } + final Path pathCacheInfo = getCachedInfoPath(); + log.info("Cache folder {}", cacheFolder); + try { + if (Files.exists(pathCacheInfo)) { + ObjectMapper mapper = new ObjectMapper(); + final JsonNode jsonNode = mapper.readTree(pathCacheInfo.toFile()); + + // Read the files array + TypeReference> typeRef = new TypeReference>() { + }; + final List files = mapper + .readValue(mapper.treeAsTokens(jsonNode.get("files")), typeRef); + + // Update the map with the remote url as a key and the file cache info as a value + cachedFiles.putAll(Collections + .synchronizedMap(files + .stream() + .filter(FileCached::exists) + .collect(Collectors.toMap(FileCached::getRemoteURL, Function.identity())) + ) + ); + log.info("Number of file already in the cache {}", cachedFiles.size()); + + } + } catch (Exception e) { + log.error("Cannot initialized the cache", e); + } + } + + public static Optional getFileCached(final URL remoteURL) + throws URISyntaxException, NoSuchMethodException, ScriptException, + IOException { + return getFileCached(remoteURL, true); + } + + public static Optional getFileCached(final URL remoteURL, boolean enableCache) + throws URISyntaxException, NoSuchMethodException, ScriptException, + IOException { + // Return always and empty file if the cache is not enable + if (!(enableCache && FileDownloaderCache.enableCache)) { + log.info("The cache is not enable."); + return Optional.empty(); + } + final String[] splitPath = remoteURL.getPath().split("/"); + if (splitPath.length == 0) { + log.warn("The remote path as no file name {}", remoteURL); + return Optional.empty(); + } + // Create the path where the cached file should exist + final Deque addFirstRemoteURL = new LinkedList<>(Arrays.asList(splitPath)); + addFirstRemoteURL.addFirst(remoteURL.getHost()); + final Path cacheFilePath = Paths.get(cacheFolder, addFirstRemoteURL.toArray(new String[0])); + + // Take from the cache the file info or build from scratch + final FileCached fileCached = Optional.ofNullable(cachedFiles.get(remoteURL.toString())) + .orElseGet(() -> new FileCached(remoteURL.toString(), cacheFilePath.toString())); + + // If the file is change of the cache is disable run the HEAD request to check if the file is changed + if (fileCached.isExpire() || !fileCached.exists()) { + // Update remote etag and cache control header + final Optional fileCachedInfoUpdated = + FileDownloaderCache.updateCacheInfo(remoteURL, (remoteETagClean, cacheControl) -> { + // Check cache control data + if (cacheControl.isNoCache() || cacheControl.isMustRevalidate() || cacheControl.isNoStore()) { + log.warn("The file {} must not be cache due to cache control header {}", + remoteURL, cacheControl); + return Optional.empty(); + } + log.info("Update cached info of {}, createdAt {}, previous eTag {}, last eTag {}, cache control header {} ", + remoteURL, fileCached.createdAt, fileCached.eTag, remoteETagClean, cacheControl); + final FileCached fileCachedUpdateETag = new FileCached( + remoteURL.toString(), + cacheFilePath.toString(), + fileCached.eTag, + remoteETagClean, // Set the lastETag + fileCached.md5, + cacheControl // Set the new cache control + ); + cachedFiles.put(remoteURL.toString(), fileCachedUpdateETag); + return Optional.of(fileCachedUpdateETag); + }); + FileDownloaderCache.updateCacheFilesInfo(); + return fileCachedInfoUpdated; + } + return Optional.of(fileCached); + } + + private static Optional updateCacheInfo(URL remoteURL, BiFunction> getNewFile) + throws URISyntaxException, NoSuchMethodException, ScriptException, + IOException { + // Update the headers of the cached file + final HttpURLConnection headRequest = new HttpConnectionManager( + remoteURL).makeConnection((connection) -> { + try { + connection.setRequestMethod("HEAD"); + } catch (ProtocolException e) { + log.error("Invalid protocol", e); + } + }); + final int responseCode = headRequest.getResponseCode(); + headRequest.disconnect(); + // Something bad is happening return a conservative true to try to download the file + if (responseCode < 200 || responseCode >= 300) { + log.warn("The head request return a bad response code " + responseCode); + // if something bad happend + return Optional.empty(); + } + // Get all the useful headers + String remoteETag = headRequest.getHeaderField("ETag"); + String cacheControlHeader = headRequest.getHeaderField("Cache-Control"); + if (remoteETag != null && cacheControlHeader != null) { + final String remoteETagClean = remoteETag.trim().replace("\"", ""); + final CacheControl cacheControl = CacheControl.valueOf(cacheControlHeader); + return getNewFile.apply(remoteETagClean, cacheControl); + } + log.warn("The head request do not return the ETag {} or the Cache-Control {}", remoteETag, cacheControlHeader); + return Optional.empty(); + } + + private synchronized static void updateCacheFilesInfo() throws IOException { + ObjectMapper mapper = new ObjectMapper(); + // Generate a pretty json + mapper.enable(SerializationFeature.INDENT_OUTPUT); + final ObjectNode objectNode = mapper.createObjectNode(); + // Generate a json {"files":[...{files_info}...]} + objectNode.putArray("files").addAll( + cachedFiles.values().stream() + .map((v) -> mapper.convertValue(v, JsonNode.class)) + .collect(Collectors.toList())); + // Create the path Arduino15/cache + Path cachedFileInfo = getCachedInfoPath(); + if (Files.notExists(cachedFileInfo)) { + Files.createDirectories(cachedFileInfo.getParent()); + } + log.info("Update cache file info in {}, number of cached files is {}", cachedFileInfo.toFile(), cachedFiles.size()); + // Write to cache.json + mapper.writeValue(cachedFileInfo.toFile(), objectNode); + } + + private static Path getCachedInfoPath() { + return Paths.get(cacheFolder, "cache.json"); + } + + @JsonIgnoreProperties(ignoreUnknown = true) + static class FileCached { + private final String remoteURL; + private final String localPath; + private final String eTag; + private final String lastETag; + private final String md5; + private final String createdAt; + private final CacheControl cacheControl; + + FileCached() { + this.remoteURL = null; + this.localPath = null; + lastETag = null; + eTag = null; + md5 = null; + createdAt = null; + cacheControl = null; + } + + FileCached(String remoteURL, String localPath) { + this.remoteURL = remoteURL; + this.localPath = localPath; + lastETag = null; + eTag = null; + md5 = null; + createdAt = LocalDateTime.now().format(DateTimeFormatter.ISO_DATE_TIME); + cacheControl = null; + } + + public FileCached(String remoteURL, String localPath, String eTag, String lastETag, String md5, CacheControl cacheControl) { + this.remoteURL = remoteURL; + this.localPath = localPath; + this.eTag = eTag; + this.lastETag = lastETag; + this.md5 = md5; + this.createdAt = LocalDateTime.now().format(DateTimeFormatter.ISO_DATE_TIME); + this.cacheControl = cacheControl; + } + + @JsonIgnore + public boolean isExpire() { + // Check if the file is expire + final LocalDateTime now = LocalDateTime.now(); + return this.getExpiresTime().isBefore(now) || this.getExpiresTime().isEqual(now); + } + + @JsonIgnore + public boolean isNotChange() { + return !isChange(); + } + + @JsonIgnore + public boolean isChange() { + // Check if the file is expire + if (!isExpire()) { + log.debug("The file \"{}\" is no expire, the eTag will not be checked. Expire time: {}", localPath, + this.getExpiresTime().format(DateTimeFormatter.ISO_DATE_TIME)); + return false; + } + + if (lastETag != null) { + // If are different means that the file is change + return !lastETag.equals(eTag); + } + return true; + } + + @JsonIgnore + public boolean exists() { + return localPath != null && Files.exists(Paths.get(localPath)); + } + + @JsonIgnore + public Optional getFileFromCache() { + if (md5Check()) { + return Optional.of(Paths.get(localPath).toFile()); + } + return Optional.empty(); + + } + + public synchronized void updateCacheFile(File fileToCache) throws Exception { + Path cacheFilePath = Paths.get(localPath); + + // If the cache directory does not exist create it + if (!Files.exists(cacheFilePath.getParent())) { + Files.createDirectories(cacheFilePath.getParent()); + } + FileUtils.copyFile(fileToCache, cacheFilePath.toFile()); + final String md5 = this.calculateMD5(); + final String eTag; + if (lastETag == null) { + log.warn("The eTag was not calculate this time, is not the right behaviour fileCached={}, md5={}", this, md5); + eTag = this.eTag; + } else { + eTag = this.lastETag; + } + FileCached newFileCached = new FileCached( + this.remoteURL, + this.localPath, + eTag, // Initialize the right eTag with the last eTag because the file was updated + eTag, + md5, + this.cacheControl + ); + log.info("Update cache file: {}", newFileCached); + cachedFiles.put(remoteURL, newFileCached); + updateCacheFilesInfo(); + + } + + public synchronized void invalidateCache() throws IOException { + cachedFiles.remove(remoteURL); + Files.deleteIfExists(Paths.get(localPath)); + } + + private String calculateMD5() throws IOException, NoSuchAlgorithmException { + if (exists()) { + return FileHash.hash(Paths.get(localPath).toFile(), "MD5"); + } + return null; + } + + @JsonIgnore + public boolean md5Check() { + try { + return !Objects.isNull(getMD5()) && Objects.equals(calculateMD5(), getMD5()); + } catch (Exception e) { + log.error("Fail to calculate the MD5. file={}", this, e); + return false; + } + } + + @JsonIgnore + public LocalDateTime getExpiresTime() { + final int maxAge; + if (cacheControl != null) { + maxAge = cacheControl.getMaxAge(); + } else { + maxAge = 0; + } + if (createdAt != null) { + return LocalDateTime.parse(createdAt, DateTimeFormatter.ISO_DATE_TIME) + .plusSeconds(maxAge); + } + return LocalDateTime.now(); + + } + + public String getExpires() { + return getExpiresTime().toString(); + } + + public String getMD5() { + return md5; + } + + public String geteTag() { + return eTag; + } + + public String getRemoteURL() { + return remoteURL; + } + + public String getLocalPath() { + return localPath; + } + + public String getCreatedAt() { + return createdAt; + } + + public CacheControl getCacheControl() { + return cacheControl; + } + + @Override + public String toString() { + return "FileCached{" + + "eTag='" + eTag + '\'' + + ", lastETag='" + lastETag + '\'' + + ", remoteURL='" + remoteURL + '\'' + + ", localPath='" + localPath + '\'' + + ", md5='" + md5 + '\'' + + ", createdAt='" + createdAt + '\'' + + ", cacheControl=" + cacheControl + + '}'; + } + } +} diff --git a/arduino-core/src/cc/arduino/utils/network/HttpConnectionManager.java b/arduino-core/src/cc/arduino/utils/network/HttpConnectionManager.java new file mode 100644 index 00000000000..25577e0dcd2 --- /dev/null +++ b/arduino-core/src/cc/arduino/utils/network/HttpConnectionManager.java @@ -0,0 +1,168 @@ +/* + * This file is part of Arduino. + * + * Copyright 2019 Arduino LLC (http://www.arduino.cc/) + * + * Arduino is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As a special exception, you may use this file as part of a free software + * library without restriction. Specifically, if other files instantiate + * templates or use macros or inline functions from this file, or you compile + * this file and link it with other files to produce an executable, this + * file does not by itself cause the resulting executable to be covered by + * the GNU General Public License. This exception does not however + * invalidate any other reasons why the executable file might be covered by + * the GNU General Public License. + */ + +package cc.arduino.utils.network; + +import cc.arduino.net.CustomProxySelector; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import processing.app.BaseNoGui; +import processing.app.PreferencesData; + +import javax.script.ScriptException; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.Proxy; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.UUID; +import java.util.function.Consumer; + +public class HttpConnectionManager { + private static Logger log = LogManager.getLogger(HttpConnectionManager.class); + private static final String userAgent; + private static final int connectTimeout; + private static final int maxRedirectNumber; + private final URL requestURL; + private final String id; + + + static { + final String defaultUserAgent = String.format( + "ArduinoIDE/%s (%s; %s; %s; %s) Java/%s (%s)", + BaseNoGui.VERSION_NAME, + System.getProperty("os.name"), + System.getProperty("os.version"), + System.getProperty("os.arch"), + System.getProperty("user.language"), + System.getProperty("java.version"), + System.getProperty("java.vendor") + ); + userAgent = PreferencesData.get("http.user_agent", defaultUserAgent); + int connectTimeoutFromConfig = 5000; + try { + connectTimeoutFromConfig = + Integer.parseInt( + PreferencesData.get("http.connection_timeout_ms", "5000")); + } catch (NumberFormatException e) { + log.warn( + "Cannot parse the http.connection_timeout configuration switch to default {} milliseconds", connectTimeoutFromConfig, e.getCause()); + } + connectTimeout = connectTimeoutFromConfig; + // Set by default 20 max redirect to follow + int maxRedirectNumberConfig = 20; + try { + maxRedirectNumberConfig = + Integer.parseInt( + PreferencesData.get("http.max_redirect_number", "20")); + } catch (NumberFormatException e) { + log.warn( + "Cannot parse the http.max_redirect_number configuration switch to default {}", maxRedirectNumberConfig, e.getCause()); + } + maxRedirectNumber = maxRedirectNumberConfig; + } + + public HttpConnectionManager(URL requestURL) { + this.requestURL = requestURL; + if (requestURL.getHost().endsWith("arduino.cc")) { + final String idString = PreferencesData.get("update.id", "0"); + id = Long.toString(Long.parseLong(idString)); + } else { + id = null; + } + + } + + public HttpURLConnection makeConnection(Consumer beforeConnection) + throws IOException, NoSuchMethodException, ScriptException, URISyntaxException { + return makeConnection(this.requestURL, 0, beforeConnection); + } + + + public HttpURLConnection makeConnection() + throws IOException, NoSuchMethodException, ScriptException, URISyntaxException { + return makeConnection(this.requestURL, 0, (c) -> { + }); + } + + private HttpURLConnection makeConnection(URL requestURL, int movedTimes, + Consumer beforeConnection) throws IOException, URISyntaxException, ScriptException, NoSuchMethodException { + if (movedTimes > maxRedirectNumber) { + log.warn("Too many redirect " + requestURL); + throw new IOException("Too many redirect " + requestURL); + } + + Proxy proxy = new CustomProxySelector(PreferencesData.getMap()) + .getProxyFor(requestURL.toURI()); + log.debug("Using proxy {}", proxy); + + final String requestId = UUID.randomUUID().toString() + .toUpperCase().replace("-", "").substring(0, 16); + HttpURLConnection connection = (HttpURLConnection) requestURL + .openConnection(proxy); + connection.setRequestProperty("User-agent", userAgent); + connection.setRequestProperty("X-Request-ID", requestId); + if (id != null) { + connection.setRequestProperty("X-ID", id); + } + if (requestURL.getUserInfo() != null) { + String auth = "Basic " + new String( + new Base64().encode(requestURL.getUserInfo().getBytes())); + connection.setRequestProperty("Authorization", auth); + } + + int initialSize = 0; + connection.setRequestProperty("Range", "bytes=" + initialSize + "-"); + connection.setConnectTimeout(connectTimeout); + beforeConnection.accept(connection); + + // Connect + log.info("Connect to {}, method={}, request id={}", requestURL, connection.getRequestMethod(), requestId); + + connection.connect(); + int resp = connection.getResponseCode(); + log.info("Request complete URL=\"{}\", method={}, response code={}, request id={}, headers={}", + requestURL, connection.getRequestMethod(), resp, requestId, StringUtils.join(connection.getHeaderFields())); + + if (resp == HttpURLConnection.HTTP_MOVED_PERM + || resp == HttpURLConnection.HTTP_MOVED_TEMP) { + + URL newUrl = new URL(connection.getHeaderField("Location")); + log.info("The response code was a 301,302 so try again with the new URL " + newUrl); + + return this.makeConnection(newUrl, movedTimes + 1, beforeConnection); + } + + return connection; + } + +} + diff --git a/arduino-core/src/processing/app/Platform.java b/arduino-core/src/processing/app/Platform.java index 535c31ed7d3..5e80563649a 100644 --- a/arduino-core/src/processing/app/Platform.java +++ b/arduino-core/src/processing/app/Platform.java @@ -23,6 +23,11 @@ package processing.app; import cc.arduino.packages.BoardPort; +import cc.arduino.utils.network.HttpConnectionManager; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import processing.app.debug.TargetBoard; import processing.app.debug.TargetPackage; import processing.app.debug.TargetPlatform; @@ -31,19 +36,10 @@ import javax.swing.*; import java.io.File; import java.io.IOException; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.ArrayList; -import java.util.Arrays; - -import java.net.URL; -import java.net.URLConnection; -import java.net.HttpURLConnection; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.DeserializationFeature; import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.*; import static processing.app.I18n.tr; @@ -64,7 +60,7 @@ * know if name is proper Java package syntax.) */ public class Platform { - + private static Logger log = LogManager.getLogger(Platform.class); /** * Set the default L & F. While I enjoy the bounty of the sixteen possible @@ -170,6 +166,7 @@ private static void loadLib(File lib) { } private native String resolveDeviceAttachedToNative(String serial); + private native String[] listSerialsNative(); public String preListAllCandidateDevices() { @@ -180,7 +177,7 @@ public List listSerials() { return new ArrayList<>(Arrays.asList(listSerialsNative())); } - public List listSerialsNames(){ + public List listSerialsNames() { List list = new LinkedList<>(); for (String port : listSerialsNative()) { list.add(port.split("_")[0]); @@ -188,46 +185,33 @@ public List listSerialsNames(){ return list; } - public static class BoardCloudAPIid { - public BoardCloudAPIid() { } - private String name; - private String architecture; - private String id; - public String getName() { return name; } - public String getArchitecture() { return architecture; } - public String getId() { return id; } - public void setName(String tmp) { name = tmp; } - public void setArchitecture(String tmp) { architecture = tmp; } - public void setId(String tmp) { id = tmp; } - } - public synchronized void getBoardWithMatchingVidPidFromCloud(String vid, String pid) { // this method is less useful in Windows < WIN10 since you need drivers to be already installed ObjectMapper mapper = new ObjectMapper(); mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); try { - URL jsonUrl = new URL("http", "api-builder.arduino.cc", 80, "/builder/v1/boards/0x"+vid+"/0x"+pid); - URLConnection connection = jsonUrl.openConnection(); - String userAgent = "ArduinoIDE/" + BaseNoGui.VERSION_NAME + " Java/" - + System.getProperty("java.version"); - connection.setRequestProperty("User-agent", userAgent); - connection.connect(); - HttpURLConnection httpConnection = (HttpURLConnection) connection; + URL jsonUrl = new URL(String.format("https://builder.arduino.cc/builder/v1/boards/0x%s/0x%s", vid, pid)); + + final HttpURLConnection httpConnection = new HttpConnectionManager(jsonUrl) + .makeConnection(); int code = httpConnection.getResponseCode(); if (code == 404) { + log.warn("Fail to get the Vid Pid information from the builder response code={}", code); return; } InputStream is = httpConnection.getInputStream(); BoardCloudAPIid board = mapper.readValue(is, BoardCloudAPIid.class); + log.info("Board info from the cloud {}", board); // Launch a popup with a link to boardmanager#board.getName() // replace spaces with & String realBoardName = board.getName().replaceAll("\\(.*?\\)", "").trim(); String boardNameReplaced = realBoardName.replaceAll(" ", "&"); - String message = I18n.format(tr("{0}Install this package{1} to use your {2} board"), "", "", realBoardName); + String message = I18n.format(tr("{0}Install this package{1} to use your {2} board"), "", "", realBoardName); BaseNoGui.setBoardManagerLink(message); } catch (Exception e) { // No connection no problem, fail silently //e.printStackTrace(); + } } @@ -259,7 +243,7 @@ public synchronized Map resolveDeviceByVendorIdProductId(String boardData.put("board", board); boardData.put("vid", vids.get(i)); boardData.put("pid", pids.get(i)); - String extrafields = vid_pid_iSerial.substring(vidPid.length()+1); + String extrafields = vid_pid_iSerial.substring(vidPid.length() + 1); String[] parts = extrafields.split("_"); boardData.put("iserial", parts[0]); return boardData; @@ -272,6 +256,56 @@ public synchronized Map resolveDeviceByVendorIdProductId(String return null; } + public static class BoardCloudAPIid { + + private String fqbn; + private String name; + private String architecture; + private String id; + + public String getName() { + return name; + } + + public String getFqbn() { + return fqbn; + } + + public String getArchitecture() { + return architecture; + } + + public String getId() { + return id; + } + + public void setName(String tmp) { + name = tmp; + } + + public void setFqbn(String fqbn) { + this.fqbn = fqbn; + } + + public void setArchitecture(String tmp) { + architecture = tmp; + } + + public void setId(String tmp) { + id = tmp; + } + + @Override + public String toString() { + return "BoardCloudAPIid{" + + "name='" + name + '\'' + + ", fqbn='" + fqbn + '\'' + + ", architecture='" + architecture + '\'' + + ", id='" + id + '\'' + + '}'; + } + } + public String resolveDeviceByBoardID(Map packages, String boardId) { assert packages != null; assert boardId != null; diff --git a/arduino-core/src/processing/app/PreferencesData.java b/arduino-core/src/processing/app/PreferencesData.java index 8ab4852b0b9..251922c0e25 100644 --- a/arduino-core/src/processing/app/PreferencesData.java +++ b/arduino-core/src/processing/app/PreferencesData.java @@ -265,7 +265,10 @@ static public Font getFont(String attr) { } public static Collection getCollection(String key) { - return Arrays.asList(get(key, "").split(",")); + return Arrays.stream(get(key, "").split(",")) + // Remove empty strings from the collection + .filter((v) -> !v.trim().isEmpty()) + .collect(Collectors.toList()); } public static void setCollection(String key, Collection values) { diff --git a/build/build.xml b/build/build.xml index 6ba0b24a4bb..4210924619d 100644 --- a/build/build.xml +++ b/build/build.xml @@ -438,6 +438,7 @@