Split download and check signature, add check signature to library index

Author: Mattia Bertorello

Diff for: arduino-core/src/cc/arduino/contributions/DownloadableContributionsDownloader.java

@@ -33,20 +33,22 @@
 import cc.arduino.utils.MultiStepProgress;
 import cc.arduino.utils.Progress;
 import cc.arduino.utils.network.FileDownloader;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import processing.app.BaseNoGui;
+import processing.app.PreferencesData;
 
 import java.io.File;
 import java.net.URL;
 import java.nio.file.*;
+import java.util.LinkedList;
 import java.util.List;
 
 import static processing.app.I18n.format;
 import static processing.app.I18n.tr;
 
 public class DownloadableContributionsDownloader {
-  private static Logger log = LoggerFactory.getLogger(DownloadableContributionsDownloader.class);
+  private static Logger log = LogManager.getLogger(DownloadableContributionsDownloader.class);
 
   private final File stagingFolder;
 
@@ -147,55 +149,71 @@ public void downloadIndexAndSignature(MultiStepProgress progress, List<String> d
 
     // Extract the file name from the url
     URL packageIndexUrl = new URL(packageIndexUrlString);
-    URL packageIndexSignatureUrl = new URL(packageIndexUrlString + ".sig");
     String[] urlPathParts = packageIndexUrl.getFile().split("/");
     File packageIndex = BaseNoGui.indexer.getIndexFile(urlPathParts[urlPathParts.length - 1]);
-    // Signature file name
-    File packageIndexSignature = BaseNoGui.indexer.getIndexFile(urlPathParts[urlPathParts.length - 1] + ".sig");
 
     final String statusText = tr("Downloading platforms index...");
     downloadedFilesAccumulator.add(packageIndex.getName());
 
     // Create temp files
     File packageIndexTemp = File.createTempFile(packageIndexUrl.getPath(), ".tmp");
-    File packageIndexSignatureTemp = File.createTempFile(packageIndexSignatureUrl.getPath(), ".tmp");
     try {
       // Download package index
       download(packageIndexUrl, packageIndexTemp, progress, statusText, progressListener, true);
-      try {
-        // Download signature
-        download(packageIndexSignatureUrl, packageIndexSignatureTemp, progress, statusText, progressListener, true);
-
-        // Verify the signature before move the files
-        boolean signatureVerified = signatureVerifier.isSigned(packageIndexTemp, packageIndexSignatureTemp);
-        if (signatureVerified) {
-          // Move if the signature is ok
-          Files.move(packageIndexTemp.toPath(), packageIndex.toPath(), StandardCopyOption.REPLACE_EXISTING);
-          Files.move(packageIndexSignatureTemp.toPath(), packageIndexSignature.toPath(), StandardCopyOption.REPLACE_EXISTING);
-          downloadedFilesAccumulator.add(packageIndexSignature.getName());
-        } else {
-          downloadedFilesAccumulator.remove(packageIndex.getName());
-          log.error("{} file signature verification failed. File ignored.", packageIndexSignatureUrl);
-          System.err.println(format(tr("{0} file signature verification failed. File ignored."), packageIndexUrlString));
+      final List<String> domain = new LinkedList<>(PreferencesData.getCollection("http.signature_verify_domains"));
+      // Default domain
+      domain.add("downloads.arduino.cc");
 
-        }
-      } catch (Exception e) {
-        log.error("Cannot download the signature from {} the package will be install in any case", packageIndexSignatureUrl, e);
-        if (packageIndexTemp.length() > 0) {
+      if (domain.contains(packageIndexUrl.getHost())) {
+        URL signatureUrl = new URL(packageIndexUrl.toString() + ".sig");
+
+        if (checkSignature(progress, downloadedFilesAccumulator, signatureUrl, progressListener, signatureVerifier, statusText, packageIndexTemp)) {
           Files.move(packageIndexTemp.toPath(), packageIndex.toPath(), StandardCopyOption.REPLACE_EXISTING);
         } else {
-          log.error("The temporarily package index file is empty (path:{},url:{}), It cannot be move there {} ",
-            packageIndexTemp.toPath(), packageIndexUrlString, packageIndex.toPath());
+          downloadedFilesAccumulator.remove(packageIndex.getName());
         }
+      } else {
+        log.info("The domain is not selected to verify the signature. domain list: {}, packageIndex: {}", domain, packageIndexUrl);
       }
-
     } catch (Exception e) {
       downloadedFilesAccumulator.remove(packageIndex.getName());
       throw e;
     } finally {
       // Delete useless temp file
       Files.deleteIfExists(packageIndexTemp.toPath());
+    }
+  }
+
+  public boolean checkSignature(MultiStepProgress progress, List<String> downloadedFilesAccumulator, URL signatureUrl, ProgressListener progressListener, SignatureVerifier signatureVerifier, String statusText, File fileToVerify) throws Exception {
+
+    File packageIndexSignatureTemp = File.createTempFile(signatureUrl.getPath(), ".tmp");
+    // Signature file name
+    String[] urlPathParts = signatureUrl.getFile().split("/");
+    File packageIndexSignature = BaseNoGui.indexer.getIndexFile(urlPathParts[urlPathParts.length - 1]);
+
+    try {
+      // Download signature
+      download(signatureUrl, packageIndexSignatureTemp, progress, statusText, progressListener, true);
+
+      // Verify the signature before move the files
+      boolean signatureVerified = signatureVerifier.isSigned(fileToVerify, packageIndexSignatureTemp);
+      if (signatureVerified) {
+        log.info("Signature verified. url={}, signature url={}, file to verify={}, signature file={}", signatureUrl, signatureUrl, fileToVerify, packageIndexSignatureTemp);
+        // Move if the signature is ok
+        Files.move(packageIndexSignatureTemp.toPath(), packageIndexSignature.toPath(), StandardCopyOption.REPLACE_EXISTING);
+        downloadedFilesAccumulator.add(packageIndexSignature.getName());
+      } else {
+        log.error("{} file signature verification failed. File ignored.", signatureUrl);
+        System.err.println(format(tr("{0} file signature verification failed. File ignored."), signatureUrl.toString()));
+      }
+      return signatureVerified;
+    } catch (Exception e) {
+      log.error("Cannot download the signature from {} the package will be discard", signatureUrl, e);
+      throw e;
+    } finally {
       Files.deleteIfExists(packageIndexSignatureTemp.toPath());
     }
+
   }
+
 }

Diff for: arduino-core/src/cc/arduino/contributions/libraries/LibraryInstaller.java

@@ -31,10 +31,13 @@
 
 import cc.arduino.Constants;
 import cc.arduino.contributions.DownloadableContributionsDownloader;
+import cc.arduino.contributions.GPGDetachedSignatureVerifier;
 import cc.arduino.contributions.GZippedJsonDownloader;
 import cc.arduino.contributions.ProgressListener;
 import cc.arduino.utils.ArchiveExtractor;
 import cc.arduino.utils.MultiStepProgress;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import processing.app.BaseNoGui;
 import processing.app.I18n;
 import processing.app.Platform;
@@ -45,47 +48,59 @@
 import java.net.URL;
 import java.nio.file.Files;
 import java.nio.file.StandardCopyOption;
+import java.util.LinkedList;
+import java.util.List;
 import java.util.Optional;
 
 import static processing.app.I18n.tr;
 
 public class LibraryInstaller {
+  private static Logger log = LogManager.getLogger(LibraryInstaller.class);
 
   private final Platform platform;
+  private final GPGDetachedSignatureVerifier signatureVerifier;
 
-  public LibraryInstaller(Platform platform) {
+  public LibraryInstaller(Platform platform, GPGDetachedSignatureVerifier signatureVerifier) {
     this.platform = platform;
+    this.signatureVerifier = signatureVerifier;
   }
 
   public synchronized void updateIndex(ProgressListener progressListener) throws Exception {
     final MultiStepProgress progress = new MultiStepProgress(3);
 
+    List<String> downloadedFilesAccumulator = new LinkedList<>();
+
     DownloadableContributionsDownloader downloader = new DownloadableContributionsDownloader(BaseNoGui.librariesIndexer.getStagingFolder());
     // Step 1: Download index
     File outputFile = BaseNoGui.librariesIndexer.getIndexFile();
     // Create temp files
     File libraryIndexTemp = File.createTempFile(new URL(Constants.LIBRARY_INDEX_URL).getPath(), ".tmp");
+    final URL libraryURL = new URL(Constants.LIBRARY_INDEX_URL);
+    final String statusText = tr("Downloading libraries index...");
     try {
-      GZippedJsonDownloader gZippedJsonDownloader = new GZippedJsonDownloader(downloader, new URL(Constants.LIBRARY_INDEX_URL), new URL(Constants.LIBRARY_INDEX_URL_GZ));
-      gZippedJsonDownloader.download(libraryIndexTemp, progress, tr("Downloading libraries index..."), progressListener);
+      GZippedJsonDownloader gZippedJsonDownloader = new GZippedJsonDownloader(downloader, libraryURL, new URL(Constants.LIBRARY_INDEX_URL_GZ));
+      gZippedJsonDownloader.download(libraryIndexTemp, progress, statusText, progressListener);
     } catch (InterruptedException e) {
       // Download interrupted... just exit
       return;
     }
     progress.stepDone();
 
-    // TODO: Check downloaded index
-
-    // Replace old index with the updated one
-    if (libraryIndexTemp.length() > 0) {
-      Files.move(libraryIndexTemp.toPath(), outputFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
-    }
+    URL signatureUrl = new URL(libraryURL.toString() + ".sig");
+    if (downloader.checkSignature(progress, downloadedFilesAccumulator, signatureUrl, progressListener, signatureVerifier, statusText, libraryIndexTemp)) {
+      // Replace old index with the updated one
+      if (libraryIndexTemp.length() > 0) {
+        Files.move(libraryIndexTemp.toPath(), outputFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
+      }
 
-    // Step 2: Parse index
-    BaseNoGui.librariesIndexer.parseIndex();
+      // Step 2: Parse index
+      BaseNoGui.librariesIndexer.parseIndex();
 
-    // Step 3: Rescan index
-    rescanLibraryIndex(progress, progressListener);
+      // Step 3: Rescan index
+      rescanLibraryIndex(progress, progressListener);
+    } else {
+      log.error("Fail to verify the signature of {}", libraryURL);
+    }
   }
 
   public synchronized void install(ContributedLibrary lib, Optional<ContributedLibrary> mayReplacedLib, ProgressListener progressListener) throws Exception {

Diff for: arduino-core/src/cc/arduino/contributions/packages/ContributionInstaller.java

@@ -41,8 +41,8 @@
 import org.apache.commons.exec.DefaultExecutor;
 import org.apache.commons.exec.Executor;
 import org.apache.commons.exec.PumpStreamHandler;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import processing.app.BaseNoGui;
 import processing.app.I18n;
 import processing.app.Platform;
@@ -63,7 +63,7 @@
 import static processing.app.I18n.tr;
 
 public class ContributionInstaller {
-  private static Logger log = LoggerFactory.getLogger(ContributionInstaller.class);
+  private static Logger log = LogManager.getLogger(ContributionInstaller.class);
 
   private final Platform platform;
   private final SignatureVerifier signatureVerifier;
@@ -267,10 +267,11 @@ public synchronized List<String> remove(ContributedPlatform contributedPlatform)
       // now try to remove the containing TOOL_NAME folder
       // (and silently fail if another version of the tool is installed)
       try {
-        Files.deleteIfExists(destFolder.getParentFile().toPath());
+        Files.delete(destFolder.getParentFile().toPath());
       } catch (Exception e) {
         // ignore
-        log.error(e.getMessage(), e);
+        log.info("The directory is not empty there is another version installed. directory {}",
+          destFolder.getParentFile().toPath(),  e);
       }
     }