001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.io; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.io.BufferedReader; 007import java.io.ByteArrayOutputStream; 008import java.io.Closeable; 009import java.io.File; 010import java.io.IOException; 011import java.io.InputStream; 012import java.math.BigInteger; 013import java.net.HttpURLConnection; 014import java.net.MalformedURLException; 015import java.net.URL; 016import java.nio.charset.StandardCharsets; 017import java.nio.file.Files; 018import java.nio.file.StandardCopyOption; 019import java.security.MessageDigest; 020import java.security.NoSuchAlgorithmException; 021import java.util.ArrayList; 022import java.util.Arrays; 023import java.util.Enumeration; 024import java.util.List; 025import java.util.Map; 026import java.util.concurrent.ConcurrentHashMap; 027import java.util.concurrent.TimeUnit; 028import java.util.zip.ZipEntry; 029import java.util.zip.ZipFile; 030 031import org.openstreetmap.josm.Main; 032import org.openstreetmap.josm.spi.preferences.Config; 033import org.openstreetmap.josm.tools.HttpClient; 034import org.openstreetmap.josm.tools.Logging; 035import org.openstreetmap.josm.tools.Pair; 036import org.openstreetmap.josm.tools.Utils; 037 038/** 039 * Downloads a file and caches it on disk in order to reduce network load. 040 * 041 * Supports URLs, local files, and a custom scheme (<code>resource:</code>) to get 042 * resources from the current *.jar file. (Local caching is only done for URLs.) 043 * <p> 044 * The mirrored file is only downloaded if it has been more than 7 days since 045 * last download. (Time can be configured.) 046 * <p> 047 * The file content is normally accessed with {@link #getInputStream()}, but 048 * you can also get the mirrored copy with {@link #getFile()}. 049 */ 050public class CachedFile implements Closeable { 051 052 /** 053 * Caching strategy. 054 */ 055 public enum CachingStrategy { 056 /** 057 * If cached file on disk is older than a certain time (7 days by default), 058 * consider the cache stale and try to download the file again. 059 */ 060 MaxAge, 061 /** 062 * Similar to MaxAge, considers the cache stale when a certain age is 063 * exceeded. In addition, a If-Modified-Since HTTP header is added. 064 * When the server replies "304 Not Modified", this is considered the same 065 * as a full download. 066 */ 067 IfModifiedSince 068 } 069 070 protected String name; 071 protected long maxAge; 072 protected String destDir; 073 protected String httpAccept; 074 protected CachingStrategy cachingStrategy; 075 076 private boolean fastFail; 077 private HttpClient activeConnection; 078 protected File cacheFile; 079 protected boolean initialized; 080 protected String parameter; 081 082 public static final long DEFAULT_MAXTIME = -1L; 083 public static final long DAYS = TimeUnit.DAYS.toSeconds(1); // factor to get caching time in days 084 085 private final Map<String, String> httpHeaders = new ConcurrentHashMap<>(); 086 087 /** 088 * Constructs a CachedFile object from a given filename, URL or internal resource. 089 * 090 * @param name can be:<ul> 091 * <li>relative or absolute file name</li> 092 * <li>{@code file:///SOME/FILE} the same as above</li> 093 * <li>{@code http://...} a URL. It will be cached on disk.</li> 094 * <li>{@code resource://SOME/FILE} file from the classpath (usually in the current *.jar)</li> 095 * <li>{@code josmdir://SOME/FILE} file inside josm user data directory (since r7058)</li> 096 * <li>{@code josmplugindir://SOME/FILE} file inside josm plugin directory (since r7834)</li></ul> 097 */ 098 public CachedFile(String name) { 099 this.name = name; 100 } 101 102 /** 103 * Set the name of the resource. 104 * @param name can be:<ul> 105 * <li>relative or absolute file name</li> 106 * <li>{@code file:///SOME/FILE} the same as above</li> 107 * <li>{@code http://...} a URL. It will be cached on disk.</li> 108 * <li>{@code resource://SOME/FILE} file from the classpath (usually in the current *.jar)</li> 109 * <li>{@code josmdir://SOME/FILE} file inside josm user data directory (since r7058)</li> 110 * <li>{@code josmplugindir://SOME/FILE} file inside josm plugin directory (since r7834)</li></ul> 111 * @return this object 112 */ 113 public CachedFile setName(String name) { 114 this.name = name; 115 return this; 116 } 117 118 /** 119 * Set maximum age of cache file. Only applies to URLs. 120 * When this time has passed after the last download of the file, the 121 * cache is considered stale and a new download will be attempted. 122 * @param maxAge the maximum cache age in seconds 123 * @return this object 124 */ 125 public CachedFile setMaxAge(long maxAge) { 126 this.maxAge = maxAge; 127 return this; 128 } 129 130 /** 131 * Set the destination directory for the cache file. Only applies to URLs. 132 * @param destDir the destination directory 133 * @return this object 134 */ 135 public CachedFile setDestDir(String destDir) { 136 this.destDir = destDir; 137 return this; 138 } 139 140 /** 141 * Set the accepted MIME types sent in the HTTP Accept header. Only applies to URLs. 142 * @param httpAccept the accepted MIME types 143 * @return this object 144 */ 145 public CachedFile setHttpAccept(String httpAccept) { 146 this.httpAccept = httpAccept; 147 return this; 148 } 149 150 /** 151 * Set the caching strategy. Only applies to URLs. 152 * @param cachingStrategy caching strategy 153 * @return this object 154 */ 155 public CachedFile setCachingStrategy(CachingStrategy cachingStrategy) { 156 this.cachingStrategy = cachingStrategy; 157 return this; 158 } 159 160 /** 161 * Sets the http headers. Only applies to URL pointing to http or https resources 162 * @param headers that should be sent together with request 163 * @return this object 164 */ 165 public CachedFile setHttpHeaders(Map<String, String> headers) { 166 this.httpHeaders.putAll(headers); 167 return this; 168 } 169 170 /** 171 * Sets whether opening HTTP connections should fail fast, i.e., whether a 172 * {@link HttpClient#setConnectTimeout(int) low connect timeout} should be used. 173 * @param fastFail whether opening HTTP connections should fail fast 174 */ 175 public void setFastFail(boolean fastFail) { 176 this.fastFail = fastFail; 177 } 178 179 /** 180 * Sets additional URL parameter (used e.g. for maps) 181 * @param parameter the URL parameter 182 * @since 13536 183 */ 184 public void setParam(String parameter) { 185 this.parameter = parameter; 186 } 187 188 public String getName() { 189 if (parameter != null) 190 return name.replaceAll("%<(.*)>", ""); 191 return name; 192 } 193 194 /** 195 * Returns maximum age of cache file. Only applies to URLs. 196 * When this time has passed after the last download of the file, the 197 * cache is considered stale and a new download will be attempted. 198 * @return the maximum cache age in seconds 199 */ 200 public long getMaxAge() { 201 return maxAge; 202 } 203 204 public String getDestDir() { 205 return destDir; 206 } 207 208 public String getHttpAccept() { 209 return httpAccept; 210 } 211 212 public CachingStrategy getCachingStrategy() { 213 return cachingStrategy; 214 } 215 216 /** 217 * Get InputStream to the requested resource. 218 * @return the InputStream 219 * @throws IOException when the resource with the given name could not be retrieved 220 */ 221 public InputStream getInputStream() throws IOException { 222 File file = getFile(); 223 if (file == null) { 224 if (name != null && name.startsWith("resource://")) { 225 InputStream is = getClass().getResourceAsStream( 226 name.substring("resource:/".length())); 227 if (is == null) 228 throw new IOException(tr("Failed to open input stream for resource ''{0}''", name)); 229 return is; 230 } else { 231 throw new IOException("No file found for: "+name); 232 } 233 } 234 return Files.newInputStream(file.toPath()); 235 } 236 237 /** 238 * Get the full content of the requested resource as a byte array. 239 * @return the full content of the requested resource as byte array 240 * @throws IOException in case of an I/O error 241 */ 242 public byte[] getByteContent() throws IOException { 243 try (InputStream is = getInputStream()) { 244 ByteArrayOutputStream buffer = new ByteArrayOutputStream(); 245 int nRead; 246 byte[] data = new byte[8192]; 247 while ((nRead = is.read(data, 0, data.length)) != -1) { 248 buffer.write(data, 0, nRead); 249 } 250 buffer.flush(); 251 return buffer.toByteArray(); 252 } 253 } 254 255 /** 256 * Returns {@link #getInputStream()} wrapped in a buffered reader. 257 * <p> 258 * Detects Unicode charset in use utilizing {@link UTFInputStreamReader}. 259 * 260 * @return buffered reader 261 * @throws IOException if any I/O error occurs 262 * @since 9411 263 */ 264 public BufferedReader getContentReader() throws IOException { 265 return new BufferedReader(UTFInputStreamReader.create(getInputStream())); 266 } 267 268 /** 269 * Get local file for the requested resource. 270 * @return The local cache file for URLs. If the resource is a local file, 271 * returns just that file. 272 * @throws IOException when the resource with the given name could not be retrieved 273 */ 274 public synchronized File getFile() throws IOException { 275 if (initialized) 276 return cacheFile; 277 initialized = true; 278 URL url; 279 try { 280 url = new URL(name); 281 if ("file".equals(url.getProtocol())) { 282 cacheFile = new File(name.substring("file:/".length() - 1)); 283 if (!cacheFile.exists()) { 284 cacheFile = new File(name.substring("file://".length() - 1)); 285 } 286 } else { 287 try { 288 cacheFile = checkLocal(url); 289 } catch (SecurityException e) { 290 throw new IOException(e); 291 } 292 } 293 } catch (MalformedURLException e) { 294 if (name == null || name.startsWith("resource://")) { 295 return null; 296 } else if (name.startsWith("josmdir://")) { 297 cacheFile = new File(Config.getDirs().getUserDataDirectory(false), name.substring("josmdir://".length())); 298 } else if (name.startsWith("josmplugindir://")) { 299 cacheFile = new File(Main.pref.getPluginsDirectory(), name.substring("josmplugindir://".length())); 300 } else { 301 cacheFile = new File(name); 302 } 303 } 304 if (cacheFile == null) 305 throw new IOException("Unable to get cache file for "+getName()); 306 return cacheFile; 307 } 308 309 /** 310 * Looks for a certain entry inside a zip file and returns the entry path. 311 * 312 * Replies a file in the top level directory of the ZIP file which has an 313 * extension <code>extension</code>. If more than one files have this 314 * extension, the last file whose name includes <code>namepart</code> 315 * is opened. 316 * 317 * @param extension the extension of the file we're looking for 318 * @param namepart the name part 319 * @return The zip entry path of the matching file. <code>null</code> if this cached file 320 * doesn't represent a zip file or if there was no matching 321 * file in the ZIP file. 322 */ 323 public String findZipEntryPath(String extension, String namepart) { 324 Pair<String, InputStream> ze = findZipEntryImpl(extension, namepart); 325 if (ze == null) return null; 326 return ze.a; 327 } 328 329 /** 330 * Like {@link #findZipEntryPath}, but returns the corresponding InputStream. 331 * @param extension the extension of the file we're looking for 332 * @param namepart the name part 333 * @return InputStream to the matching file. <code>null</code> if this cached file 334 * doesn't represent a zip file or if there was no matching 335 * file in the ZIP file. 336 * @since 6148 337 */ 338 public InputStream findZipEntryInputStream(String extension, String namepart) { 339 Pair<String, InputStream> ze = findZipEntryImpl(extension, namepart); 340 if (ze == null) return null; 341 return ze.b; 342 } 343 344 private Pair<String, InputStream> findZipEntryImpl(String extension, String namepart) { 345 File file = null; 346 try { 347 file = getFile(); 348 } catch (IOException ex) { 349 Logging.log(Logging.LEVEL_WARN, ex); 350 } 351 if (file == null) 352 return null; 353 Pair<String, InputStream> res = null; 354 try { 355 ZipFile zipFile = new ZipFile(file, StandardCharsets.UTF_8); 356 ZipEntry resentry = null; 357 Enumeration<? extends ZipEntry> entries = zipFile.entries(); 358 while (entries.hasMoreElements()) { 359 ZipEntry entry = entries.nextElement(); 360 // choose any file with correct extension. When more than one file, prefer the one which matches namepart 361 if (entry.getName().endsWith('.' + extension) && (resentry == null || entry.getName().indexOf(namepart) >= 0)) { 362 resentry = entry; 363 } 364 } 365 if (resentry != null) { 366 InputStream is = zipFile.getInputStream(resentry); 367 res = Pair.create(resentry.getName(), is); 368 } else { 369 Utils.close(zipFile); 370 } 371 } catch (IOException e) { 372 if (file.getName().endsWith(".zip")) { 373 Logging.log(Logging.LEVEL_WARN, 374 tr("Failed to open file with extension ''{2}'' and namepart ''{3}'' in zip file ''{0}''. Exception was: {1}", 375 file.getName(), e.toString(), extension, namepart), e); 376 } 377 } 378 return res; 379 } 380 381 /** 382 * Clear the cache for the given resource. 383 * This forces a fresh download. 384 * @param name the URL 385 */ 386 public static void cleanup(String name) { 387 cleanup(name, null); 388 } 389 390 /** 391 * Clear the cache for the given resource. 392 * This forces a fresh download. 393 * @param name the URL 394 * @param destDir the destination directory (see {@link #setDestDir(java.lang.String)}) 395 */ 396 public static void cleanup(String name, String destDir) { 397 URL url; 398 try { 399 url = new URL(name); 400 if (!"file".equals(url.getProtocol())) { 401 String prefKey = getPrefKey(url, destDir); 402 List<String> localPath = new ArrayList<>(Config.getPref().getList(prefKey)); 403 if (localPath.size() == 2) { 404 File lfile = new File(localPath.get(1)); 405 if (lfile.exists()) { 406 Utils.deleteFile(lfile); 407 } 408 } 409 Config.getPref().putList(prefKey, null); 410 } 411 } catch (MalformedURLException e) { 412 Logging.warn(e); 413 } 414 } 415 416 /** 417 * Get preference key to store the location and age of the cached file. 418 * 2 resources that point to the same url, but that are to be stored in different 419 * directories will not share a cache file. 420 * @param url URL 421 * @param destDir destination directory 422 * @return Preference key 423 */ 424 private static String getPrefKey(URL url, String destDir) { 425 StringBuilder prefKey = new StringBuilder("mirror."); 426 if (destDir != null) { 427 prefKey.append(destDir).append('.'); 428 } 429 prefKey.append(url.toString().replaceAll("%<(.*)>", "")); 430 return prefKey.toString().replaceAll("=", "_"); 431 } 432 433 private File checkLocal(URL url) throws IOException { 434 String prefKey = getPrefKey(url, destDir); 435 String urlStr = url.toExternalForm(); 436 if (parameter != null) 437 urlStr = urlStr.replaceAll("%<(.*)>", ""); 438 long age = 0L; 439 long maxAgeMillis = TimeUnit.SECONDS.toMillis(maxAge); 440 Long ifModifiedSince = null; 441 File localFile = null; 442 List<String> localPathEntry = new ArrayList<>(Config.getPref().getList(prefKey)); 443 boolean offline = false; 444 try { 445 checkOfflineAccess(urlStr); 446 } catch (OfflineAccessException e) { 447 Logging.trace(e); 448 offline = true; 449 } 450 if (localPathEntry.size() == 2) { 451 localFile = new File(localPathEntry.get(1)); 452 if (!localFile.exists()) { 453 localFile = null; 454 } else { 455 if (maxAge == DEFAULT_MAXTIME 456 || maxAge <= 0 // arbitrary value <= 0 is deprecated 457 ) { 458 maxAgeMillis = TimeUnit.SECONDS.toMillis(Config.getPref().getLong("mirror.maxtime", TimeUnit.DAYS.toSeconds(7))); 459 } 460 age = System.currentTimeMillis() - Long.parseLong(localPathEntry.get(0)); 461 if (offline || age < maxAgeMillis) { 462 return localFile; 463 } 464 if (cachingStrategy == CachingStrategy.IfModifiedSince) { 465 ifModifiedSince = Long.valueOf(localPathEntry.get(0)); 466 } 467 } 468 } 469 if (destDir == null) { 470 destDir = Config.getDirs().getCacheDirectory(true).getPath(); 471 } 472 473 File destDirFile = new File(destDir); 474 if (!destDirFile.exists()) { 475 Utils.mkDirs(destDirFile); 476 } 477 478 // No local file + offline => nothing to do 479 if (offline) { 480 return null; 481 } 482 483 if (parameter != null) { 484 String u = url.toExternalForm(); 485 String uc; 486 if (parameter.isEmpty()) { 487 uc = u.replaceAll("%<(.*)>", ""); 488 } else { 489 uc = u.replaceAll("%<(.*)>", "$1" + Utils.encodeUrl(parameter)); 490 } 491 if (!uc.equals(u)) 492 url = new URL(uc); 493 } 494 495 String a = urlStr.replaceAll("[^A-Za-z0-9_.-]", "_"); 496 String localPath = "mirror_" + a; 497 localPath = truncatePath(destDir, localPath); 498 destDirFile = new File(destDir, localPath + ".tmp"); 499 try { 500 activeConnection = HttpClient.create(url) 501 .setAccept(httpAccept) 502 .setIfModifiedSince(ifModifiedSince == null ? 0L : ifModifiedSince) 503 .setHeaders(httpHeaders); 504 if (fastFail) { 505 activeConnection.setReadTimeout(1000); 506 } 507 final HttpClient.Response con = activeConnection.connect(); 508 if (ifModifiedSince != null && con.getResponseCode() == HttpURLConnection.HTTP_NOT_MODIFIED) { 509 Logging.debug("304 Not Modified ({0})", urlStr); 510 if (localFile == null) 511 throw new AssertionError(); 512 Config.getPref().putList(prefKey, 513 Arrays.asList(Long.toString(System.currentTimeMillis()), localPathEntry.get(1))); 514 return localFile; 515 } else if (con.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) { 516 throw new IOException(tr("The requested URL {0} was not found", urlStr)); 517 } 518 try (InputStream is = con.getContent()) { 519 Files.copy(is, destDirFile.toPath(), StandardCopyOption.REPLACE_EXISTING); 520 } 521 activeConnection = null; 522 localFile = new File(destDir, localPath); 523 if (Main.platform.rename(destDirFile, localFile)) { 524 Config.getPref().putList(prefKey, 525 Arrays.asList(Long.toString(System.currentTimeMillis()), localFile.toString())); 526 } else { 527 Logging.warn(tr("Failed to rename file {0} to {1}.", 528 destDirFile.getPath(), localFile.getPath())); 529 } 530 } catch (IOException e) { 531 if (age >= maxAgeMillis && age < maxAgeMillis*2) { 532 Logging.warn(tr("Failed to load {0}, use cached file and retry next time: {1}", urlStr, e)); 533 return localFile; 534 } else { 535 throw e; 536 } 537 } 538 539 return localFile; 540 } 541 542 private static void checkOfflineAccess(String urlString) { 543 OnlineResource.JOSM_WEBSITE.checkOfflineAccess(urlString, Main.getJOSMWebsite()); 544 OnlineResource.OSM_API.checkOfflineAccess(urlString, OsmApi.getOsmApi().getServerUrl()); 545 } 546 547 private static String truncatePath(String directory, String fileName) { 548 if (directory.length() + fileName.length() > 255) { 549 // Windows doesn't support paths longer than 260, leave 5 chars as safe buffer, 4 will be used by ".tmp" 550 // TODO: what about filename size on other systems? 255? 551 if (directory.length() > 191 && Main.isPlatformWindows()) { 552 // digest length + name prefix == 64 553 // 255 - 64 = 191 554 // TODO: use this check only on Windows? 555 throw new IllegalArgumentException("Path " + directory + " too long to cached files"); 556 } 557 558 MessageDigest md; 559 try { 560 md = MessageDigest.getInstance("SHA-256"); 561 md.update(fileName.getBytes(StandardCharsets.UTF_8)); 562 String digest = String.format("%064x", new BigInteger(1, md.digest())); 563 return fileName.substring(0, Math.min(fileName.length(), 32)) + digest.substring(0, 32); 564 } catch (NoSuchAlgorithmException e) { 565 Logging.error(e); 566 // TODO: what better can we do here? 567 throw new IllegalArgumentException("Missing digest algorithm SHA-256", e); 568 } 569 } 570 return fileName; 571 } 572 573 /** 574 * Attempts to disconnect an URL connection. 575 * @see HttpClient#disconnect() 576 * @since 9411 577 */ 578 @Override 579 public void close() { 580 if (activeConnection != null) { 581 activeConnection.disconnect(); 582 } 583 } 584 585 /** 586 * Clears the cached file 587 * @throws IOException if any I/O error occurs 588 * @since 10993 589 */ 590 public void clear() throws IOException { 591 URL url; 592 try { 593 url = new URL(name); 594 if ("file".equals(url.getProtocol())) { 595 return; // this is local file - do not delete it 596 } 597 } catch (MalformedURLException e) { 598 return; // if it's not a URL, then it still might be a local file - better not to delete 599 } 600 File f = getFile(); 601 if (f != null && f.exists()) { 602 Utils.deleteFile(f); 603 } 604 } 605}