Skip to content

Commit 6592c42

Browse files
author
Mattia Bertorello
committed
Add the file downloader cache to make faster the library/boards manager
1 parent 00a7546 commit 6592c42

File tree

3 files changed

+258
-58
lines changed

3 files changed

+258
-58
lines changed

Diff for: ‎arduino-core/src/cc/arduino/utils/network/FileDownloader.java

+44-58
Original file line numberDiff line numberDiff line change
@@ -29,26 +29,26 @@
2929

3030
package cc.arduino.utils.network;
3131

32-
import cc.arduino.net.CustomProxySelector;
33-
import org.apache.commons.codec.binary.Base64;
3432
import org.apache.commons.compress.utils.IOUtils;
3533

3634
import processing.app.BaseNoGui;
37-
import processing.app.PreferencesData;
35+
import processing.app.helpers.FileUtils;
3836

3937
import java.io.File;
4038
import java.io.IOException;
4139
import java.io.InputStream;
4240
import java.io.RandomAccessFile;
43-
import java.net.HttpURLConnection;
44-
import java.net.Proxy;
45-
import java.net.SocketTimeoutException;
46-
import java.net.URL;
41+
import java.net.*;
4742
import java.nio.file.Files;
4843
import java.nio.file.Paths;
4944
import java.util.Observable;
45+
import java.util.Optional;
46+
import java.util.logging.Level;
47+
import java.util.logging.Logger;
5048

5149
public class FileDownloader extends Observable {
50+
private static Logger log = Logger
51+
.getLogger(FileDownloader.class.getName());
5252

5353
public enum Status {
5454
CONNECTING, //
@@ -68,15 +68,12 @@ public enum Status {
6868
private final File outputFile;
6969
private InputStream stream = null;
7070
private Exception error;
71-
private String userAgent;
7271

7372
public FileDownloader(URL url, File file) {
7473
downloadUrl = url;
7574
outputFile = file;
7675
downloaded = 0;
7776
initialSize = 0;
78-
userAgent = "ArduinoIDE/" + BaseNoGui.VERSION_NAME + " Java/"
79-
+ System.getProperty("java.version");
8077
}
8178

8279
public long getInitialSize() {
@@ -144,62 +141,49 @@ private void saveLocalFile() {
144141
}
145142

146143
private void downloadFile(boolean noResume) throws InterruptedException {
147-
RandomAccessFile file = null;
144+
RandomAccessFile randomAccessOutputFile = null;
148145

149146
try {
147+
setStatus(Status.CONNECTING);
148+
149+
final File settingsFolder = BaseNoGui.getPlatform().getSettingsFolder();
150+
final String cacheFolder = Paths.get(settingsFolder.getPath(), "cache").toString();
151+
final FileDownloaderCache fileDownloaderCache =
152+
new FileDownloaderCache(cacheFolder, downloadUrl);
153+
154+
final boolean isChanged = fileDownloaderCache.checkIfTheFileIsChanged();
155+
156+
if (!isChanged) {
157+
try {
158+
final Optional<File> fileFromCache =
159+
fileDownloaderCache.getFileFromCache();
160+
if (fileFromCache.isPresent()) {
161+
FileUtils.copyFile(fileFromCache.get(), outputFile);
162+
setStatus(Status.COMPLETE);
163+
return;
164+
}
165+
} catch (Exception e) {
166+
log.log(Level.WARNING,
167+
"Cannot get the file from the cache, will be downloaded a new one ", e.getCause());
168+
169+
}
170+
}
171+
150172
// Open file and seek to the end of it
151-
file = new RandomAccessFile(outputFile, "rw");
152-
initialSize = file.length();
173+
randomAccessOutputFile = new RandomAccessFile(outputFile, "rw");
174+
initialSize = randomAccessOutputFile.length();
153175

154176
if (noResume && initialSize > 0) {
155177
// delete file and restart downloading
156178
Files.delete(outputFile.toPath());
157179
initialSize = 0;
158180
}
159181

160-
file.seek(initialSize);
161-
162-
setStatus(Status.CONNECTING);
163-
164-
Proxy proxy = new CustomProxySelector(PreferencesData.getMap()).getProxyFor(downloadUrl.toURI());
165-
if ("true".equals(System.getProperty("DEBUG"))) {
166-
System.err.println("Using proxy " + proxy);
167-
}
168-
169-
HttpURLConnection connection = (HttpURLConnection) downloadUrl.openConnection(proxy);
170-
connection.setRequestProperty("User-agent", userAgent);
171-
if (downloadUrl.getUserInfo() != null) {
172-
String auth = "Basic " + new String(new Base64().encode(downloadUrl.getUserInfo().getBytes()));
173-
connection.setRequestProperty("Authorization", auth);
174-
}
175-
176-
connection.setRequestProperty("Range", "bytes=" + initialSize + "-");
177-
connection.setConnectTimeout(5000);
178-
setDownloaded(0);
179-
180-
// Connect
181-
connection.connect();
182-
int resp = connection.getResponseCode();
183-
184-
if (resp == HttpURLConnection.HTTP_MOVED_PERM || resp == HttpURLConnection.HTTP_MOVED_TEMP) {
185-
URL newUrl = new URL(connection.getHeaderField("Location"));
182+
randomAccessOutputFile.seek(initialSize);
186183

187-
proxy = new CustomProxySelector(PreferencesData.getMap()).getProxyFor(newUrl.toURI());
188-
189-
// open the new connnection again
190-
connection = (HttpURLConnection) newUrl.openConnection(proxy);
191-
connection.setRequestProperty("User-agent", userAgent);
192-
if (downloadUrl.getUserInfo() != null) {
193-
String auth = "Basic " + new String(new Base64().encode(downloadUrl.getUserInfo().getBytes()));
194-
connection.setRequestProperty("Authorization", auth);
195-
}
196-
197-
connection.setRequestProperty("Range", "bytes=" + initialSize + "-");
198-
connection.setConnectTimeout(5000);
199-
200-
connection.connect();
201-
resp = connection.getResponseCode();
202-
}
184+
final HttpURLConnection connection = new HttpConnectionManager(downloadUrl)
185+
.makeConnection((c) -> setDownloaded(0));
186+
final int resp = connection.getResponseCode();
203187

204188
if (resp < 200 || resp >= 300) {
205189
throw new IOException("Received invalid http status code from server: " + resp);
@@ -221,11 +205,11 @@ private void downloadFile(boolean noResume) throws InterruptedException {
221205
if (read == -1)
222206
break;
223207

224-
file.write(buffer, 0, read);
208+
randomAccessOutputFile.write(buffer, 0, read);
225209
setDownloaded(getDownloaded() + read);
226210

227211
if (Thread.interrupted()) {
228-
file.close();
212+
randomAccessOutputFile.close();
229213
throw new InterruptedException();
230214
}
231215
}
@@ -234,6 +218,8 @@ private void downloadFile(boolean noResume) throws InterruptedException {
234218
if (getDownloaded() < getDownloadSize())
235219
throw new Exception("Incomplete download");
236220
}
221+
// Set the cache whe it finish to download the file
222+
fileDownloaderCache.fillCache(outputFile);
237223
setStatus(Status.COMPLETE);
238224
} catch (InterruptedException e) {
239225
setStatus(Status.CANCELLED);
@@ -249,7 +235,7 @@ private void downloadFile(boolean noResume) throws InterruptedException {
249235
setError(e);
250236

251237
} finally {
252-
IOUtils.closeQuietly(file);
238+
IOUtils.closeQuietly(randomAccessOutputFile);
253239

254240
synchronized (this) {
255241
IOUtils.closeQuietly(stream);

Diff for: ‎arduino-core/src/cc/arduino/utils/network/FileDownloaderCache.java

+100
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
package cc.arduino.utils.network;
2+
3+
import com.sun.istack.internal.NotNull;
4+
import processing.app.PreferencesData;
5+
import processing.app.helpers.FileUtils;
6+
7+
import javax.script.ScriptException;
8+
import java.io.File;
9+
import java.io.IOException;
10+
import java.net.HttpURLConnection;
11+
import java.net.ProtocolException;
12+
import java.net.URISyntaxException;
13+
import java.net.URL;
14+
import java.nio.file.Files;
15+
import java.nio.file.Path;
16+
import java.nio.file.Paths;
17+
import java.util.Optional;
18+
import java.util.logging.Logger;
19+
20+
public class FileDownloaderCache {
21+
private static Logger log = Logger
22+
.getLogger(FileDownloaderCache.class.getName());
23+
private final String cacheFolder;
24+
private final URL remoteURL;
25+
private final Path cacheFilePath;
26+
// Will be initialized by call the checkIfTheFileIsChanged function
27+
private String eTag;
28+
29+
// BaseNoGui.getSettingsFolder()
30+
public FileDownloaderCache(@NotNull String cacheFolder, @NotNull URL remoteURL) {
31+
this.cacheFolder = cacheFolder;
32+
this.remoteURL = remoteURL;
33+
String[] splitPath = remoteURL.getPath().split("/");
34+
if (splitPath.length > 0) {
35+
this.cacheFilePath = Paths.get(cacheFolder, splitPath[splitPath.length - 1]);
36+
} else {
37+
this.cacheFilePath = null;
38+
}
39+
}
40+
41+
public boolean checkIfTheFileIsChanged()
42+
throws NoSuchMethodException, ScriptException, IOException,
43+
URISyntaxException {
44+
45+
final HttpURLConnection headRequest = new HttpConnectionManager(remoteURL)
46+
.makeConnection((connection) -> {
47+
try {
48+
connection.setRequestMethod("HEAD");
49+
} catch (ProtocolException e) {
50+
throw new RuntimeException(e);
51+
}
52+
});
53+
final int responseCode = headRequest.getResponseCode();
54+
// Something bad is happening return a conservative true to try to download the file
55+
if (responseCode < 200 || responseCode >= 300) {
56+
log.warning("The head request return a bad response code " + responseCode);
57+
return true;
58+
}
59+
60+
final String remoteETag = headRequest.getHeaderField("ETag");
61+
final String localETag = PreferencesData.get(getPreferencesDataKey());
62+
63+
// If the header doesn't exist or the local cache doesn't exist you need to download the file
64+
if (remoteETag == null || localETag == null) {
65+
return true;
66+
}
67+
eTag = remoteETag.trim().replace("\"", "");
68+
69+
// If are different means that the file is change
70+
return !eTag.equals(localETag);
71+
}
72+
73+
public Optional<File> getFileFromCache() {
74+
if (Optional.ofNullable(cacheFilePath).isPresent()) {
75+
if (Files.exists(cacheFilePath)) {
76+
return Optional.of(new File(cacheFilePath.toUri()));
77+
}
78+
}
79+
return Optional.empty();
80+
81+
}
82+
83+
public void fillCache(File fileToCache) throws Exception {
84+
if (Optional.ofNullable(eTag).isPresent() &&
85+
Optional.ofNullable(cacheFilePath).isPresent()) {
86+
87+
PreferencesData.set(getPreferencesDataKey(), eTag);
88+
// If the cache directory does not exist create it
89+
final Path cachePath = Paths.get(cacheFolder);
90+
if (!Files.exists(cachePath)) {
91+
Files.createDirectory(cachePath);
92+
}
93+
FileUtils.copyFile(fileToCache, cacheFilePath.toFile());
94+
}
95+
}
96+
97+
private String getPreferencesDataKey() {
98+
return "cache.file." + remoteURL.getPath();
99+
}
100+
}

Diff for: ‎arduino-core/src/cc/arduino/utils/network/HttpConnectionManager.java

+114
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
package cc.arduino.utils.network;
2+
3+
import cc.arduino.net.CustomProxySelector;
4+
import org.apache.commons.codec.binary.Base64;
5+
import processing.app.BaseNoGui;
6+
import processing.app.PreferencesData;
7+
8+
import javax.script.ScriptException;
9+
import java.io.IOException;
10+
import java.net.HttpURLConnection;
11+
import java.net.Proxy;
12+
import java.net.URISyntaxException;
13+
import java.net.URL;
14+
import java.util.function.Consumer;
15+
import java.util.logging.Level;
16+
import java.util.logging.Logger;
17+
18+
public class HttpConnectionManager {
19+
private static Logger log = Logger
20+
.getLogger(HttpConnectionManager.class.getName());
21+
private final URL requestURL;
22+
private final String userAgent;
23+
private int connectTimeout;
24+
25+
26+
public HttpConnectionManager(URL requestURL) {
27+
this.requestURL = requestURL;
28+
final String defaultUserAgent = String.format(
29+
"ArduinoIDE/%s (%s; %s; %s; %s) Java/%s (%s)",
30+
BaseNoGui.VERSION_NAME,
31+
System.getProperty("os.name"),
32+
System.getProperty("os.version"),
33+
System.getProperty("os.arch"),
34+
System.getProperty("user.language"),
35+
System.getProperty("java.version"),
36+
System.getProperty("java.vendor")
37+
);
38+
this.userAgent = PreferencesData.get("http.user_agent", defaultUserAgent);
39+
try {
40+
this.connectTimeout =
41+
Integer.parseInt(
42+
PreferencesData.get("http.connection_timeout", "5000"));
43+
} catch (NumberFormatException e) {
44+
log.log(Level.WARNING,
45+
"Cannot parse the http.connection_timeout configuration switch to default 5000 milliseconds", e.getCause());
46+
this.connectTimeout = 5000;
47+
}
48+
49+
}
50+
51+
public HttpURLConnection makeConnection(Consumer<HttpURLConnection> beforeConnection)
52+
throws URISyntaxException, NoSuchMethodException, IOException,
53+
ScriptException {
54+
return makeConnection(this.requestURL, 0, beforeConnection);
55+
}
56+
57+
private HttpURLConnection makeConnection(URL requestURL, int movedTimes,
58+
Consumer<HttpURLConnection> beforeConnection)
59+
throws NoSuchMethodException, ScriptException, IOException,
60+
URISyntaxException {
61+
log.info("Prepare http request to " + requestURL);
62+
if (requestURL == null) {
63+
log.warning("Invalid request url is null");
64+
throw new RuntimeException("Invalid request url is null");
65+
}
66+
if (movedTimes > 3) {
67+
log.warning("Too many redirect " + requestURL);
68+
throw new RuntimeException("Too many redirect " + requestURL);
69+
}
70+
71+
Proxy proxy = new CustomProxySelector(PreferencesData.getMap())
72+
.getProxyFor(requestURL.toURI());
73+
if ("true".equals(System.getProperty("DEBUG"))) {
74+
System.err.println("Using proxy " + proxy);
75+
}
76+
77+
HttpURLConnection connection = (HttpURLConnection) requestURL
78+
.openConnection(proxy);
79+
connection.setRequestProperty("User-agent", userAgent);
80+
if (requestURL.getUserInfo() != null) {
81+
String auth = "Basic " + new String(
82+
new Base64().encode(requestURL.getUserInfo().getBytes()));
83+
connection.setRequestProperty("Authorization", auth);
84+
}
85+
86+
int initialSize = 0;
87+
connection.setRequestProperty("Range", "bytes=" + initialSize + "-");
88+
connection.setConnectTimeout(connectTimeout);
89+
beforeConnection.accept(connection);
90+
91+
// Connect
92+
log.info("Connect to " + requestURL);
93+
94+
connection.connect();
95+
int resp = connection.getResponseCode();
96+
97+
if (resp == HttpURLConnection.HTTP_MOVED_PERM
98+
|| resp == HttpURLConnection.HTTP_MOVED_TEMP) {
99+
100+
URL newUrl = new URL(connection.getHeaderField("Location"));
101+
log.info("The response code was a 301,302 so try again with the new URL " + newUrl);
102+
103+
return this.makeConnection(newUrl, movedTimes + 1, beforeConnection);
104+
}
105+
log.info("The response code was" + resp);
106+
107+
return connection;
108+
}
109+
110+
public void setConnectTimeout(int connectTimeout) {
111+
this.connectTimeout = connectTimeout;
112+
}
113+
}
114+

0 commit comments

Comments
 (0)