001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.io; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.io.BufferedReader; 007import java.io.ByteArrayOutputStream; 008import java.io.Closeable; 009import java.io.File; 010import java.io.IOException; 011import java.io.InputStream; 012import java.net.HttpURLConnection; 013import java.net.MalformedURLException; 014import java.net.URL; 015import java.nio.charset.StandardCharsets; 016import java.nio.file.Files; 017import java.nio.file.StandardCopyOption; 018import java.util.ArrayList; 019import java.util.Arrays; 020import java.util.Enumeration; 021import java.util.List; 022import java.util.Map; 023import java.util.concurrent.ConcurrentHashMap; 024import java.util.concurrent.TimeUnit; 025import java.util.zip.ZipEntry; 026import java.util.zip.ZipFile; 027 028import org.openstreetmap.josm.Main; 029import org.openstreetmap.josm.spi.preferences.Config; 030import org.openstreetmap.josm.tools.HttpClient; 031import org.openstreetmap.josm.tools.Logging; 032import org.openstreetmap.josm.tools.Pair; 033import org.openstreetmap.josm.tools.Utils; 034 035/** 036 * Downloads a file and caches it on disk in order to reduce network load. 037 * 038 * Supports URLs, local files, and a custom scheme (<code>resource:</code>) to get 039 * resources from the current *.jar file. (Local caching is only done for URLs.) 040 * <p> 041 * The mirrored file is only downloaded if it has been more than 7 days since 042 * last download. (Time can be configured.) 043 * <p> 044 * The file content is normally accessed with {@link #getInputStream()}, but 045 * you can also get the mirrored copy with {@link #getFile()}. 046 */ 047public class CachedFile implements Closeable { 048 049 /** 050 * Caching strategy. 051 */ 052 public enum CachingStrategy { 053 /** 054 * If cached file on disk is older than a certain time (7 days by default), 055 * consider the cache stale and try to download the file again. 056 */ 057 MaxAge, 058 /** 059 * Similar to MaxAge, considers the cache stale when a certain age is 060 * exceeded. In addition, a If-Modified-Since HTTP header is added. 061 * When the server replies "304 Not Modified", this is considered the same 062 * as a full download. 063 */ 064 IfModifiedSince 065 } 066 067 protected String name; 068 protected long maxAge; 069 protected String destDir; 070 protected String httpAccept; 071 protected CachingStrategy cachingStrategy; 072 073 private boolean fastFail; 074 private HttpClient activeConnection; 075 protected File cacheFile; 076 protected boolean initialized; 077 protected String parameter; 078 079 public static final long DEFAULT_MAXTIME = -1L; 080 public static final long DAYS = TimeUnit.DAYS.toSeconds(1); // factor to get caching time in days 081 082 private final Map<String, String> httpHeaders = new ConcurrentHashMap<>(); 083 084 /** 085 * Constructs a CachedFile object from a given filename, URL or internal resource. 086 * 087 * @param name can be:<ul> 088 * <li>relative or absolute file name</li> 089 * <li>{@code file:///SOME/FILE} the same as above</li> 090 * <li>{@code http://...} a URL. It will be cached on disk.</li> 091 * <li>{@code resource://SOME/FILE} file from the classpath (usually in the current *.jar)</li> 092 * <li>{@code josmdir://SOME/FILE} file inside josm user data directory (since r7058)</li> 093 * <li>{@code josmplugindir://SOME/FILE} file inside josm plugin directory (since r7834)</li></ul> 094 */ 095 public CachedFile(String name) { 096 this.name = name; 097 } 098 099 /** 100 * Set the name of the resource. 101 * @param name can be:<ul> 102 * <li>relative or absolute file name</li> 103 * <li>{@code file:///SOME/FILE} the same as above</li> 104 * <li>{@code http://...} a URL. It will be cached on disk.</li> 105 * <li>{@code resource://SOME/FILE} file from the classpath (usually in the current *.jar)</li> 106 * <li>{@code josmdir://SOME/FILE} file inside josm user data directory (since r7058)</li> 107 * <li>{@code josmplugindir://SOME/FILE} file inside josm plugin directory (since r7834)</li></ul> 108 * @return this object 109 */ 110 public CachedFile setName(String name) { 111 this.name = name; 112 return this; 113 } 114 115 /** 116 * Set maximum age of cache file. Only applies to URLs. 117 * When this time has passed after the last download of the file, the 118 * cache is considered stale and a new download will be attempted. 119 * @param maxAge the maximum cache age in seconds 120 * @return this object 121 */ 122 public CachedFile setMaxAge(long maxAge) { 123 this.maxAge = maxAge; 124 return this; 125 } 126 127 /** 128 * Set the destination directory for the cache file. Only applies to URLs. 129 * @param destDir the destination directory 130 * @return this object 131 */ 132 public CachedFile setDestDir(String destDir) { 133 this.destDir = destDir; 134 return this; 135 } 136 137 /** 138 * Set the accepted MIME types sent in the HTTP Accept header. Only applies to URLs. 139 * @param httpAccept the accepted MIME types 140 * @return this object 141 */ 142 public CachedFile setHttpAccept(String httpAccept) { 143 this.httpAccept = httpAccept; 144 return this; 145 } 146 147 /** 148 * Set the caching strategy. Only applies to URLs. 149 * @param cachingStrategy caching strategy 150 * @return this object 151 */ 152 public CachedFile setCachingStrategy(CachingStrategy cachingStrategy) { 153 this.cachingStrategy = cachingStrategy; 154 return this; 155 } 156 157 /** 158 * Sets the http headers. Only applies to URL pointing to http or https resources 159 * @param headers that should be sent together with request 160 * @return this object 161 */ 162 public CachedFile setHttpHeaders(Map<String, String> headers) { 163 this.httpHeaders.putAll(headers); 164 return this; 165 } 166 167 /** 168 * Sets whether opening HTTP connections should fail fast, i.e., whether a 169 * {@link HttpClient#setConnectTimeout(int) low connect timeout} should be used. 170 * @param fastFail whether opening HTTP connections should fail fast 171 */ 172 public void setFastFail(boolean fastFail) { 173 this.fastFail = fastFail; 174 } 175 176 /** 177 * Sets additional URL parameter (used e.g. for maps) 178 * @param parameter the URL parameter 179 * @since 13536 180 */ 181 public void setParam(String parameter) { 182 this.parameter = parameter; 183 } 184 185 public String getName() { 186 if (parameter != null) 187 return name.replaceAll("%<(.*)>", ""); 188 return name; 189 } 190 191 /** 192 * Returns maximum age of cache file. Only applies to URLs. 193 * When this time has passed after the last download of the file, the 194 * cache is considered stale and a new download will be attempted. 195 * @return the maximum cache age in seconds 196 */ 197 public long getMaxAge() { 198 return maxAge; 199 } 200 201 public String getDestDir() { 202 return destDir; 203 } 204 205 public String getHttpAccept() { 206 return httpAccept; 207 } 208 209 public CachingStrategy getCachingStrategy() { 210 return cachingStrategy; 211 } 212 213 /** 214 * Get InputStream to the requested resource. 215 * @return the InputStream 216 * @throws IOException when the resource with the given name could not be retrieved 217 */ 218 public InputStream getInputStream() throws IOException { 219 File file = getFile(); 220 if (file == null) { 221 if (name != null && name.startsWith("resource://")) { 222 InputStream is = getClass().getResourceAsStream( 223 name.substring("resource:/".length())); 224 if (is == null) 225 throw new IOException(tr("Failed to open input stream for resource ''{0}''", name)); 226 return is; 227 } else { 228 throw new IOException("No file found for: "+name); 229 } 230 } 231 return Files.newInputStream(file.toPath()); 232 } 233 234 /** 235 * Get the full content of the requested resource as a byte array. 236 * @return the full content of the requested resource as byte array 237 * @throws IOException in case of an I/O error 238 */ 239 public byte[] getByteContent() throws IOException { 240 try (InputStream is = getInputStream()) { 241 ByteArrayOutputStream buffer = new ByteArrayOutputStream(); 242 int nRead; 243 byte[] data = new byte[8192]; 244 while ((nRead = is.read(data, 0, data.length)) != -1) { 245 buffer.write(data, 0, nRead); 246 } 247 buffer.flush(); 248 return buffer.toByteArray(); 249 } 250 } 251 252 /** 253 * Returns {@link #getInputStream()} wrapped in a buffered reader. 254 * <p> 255 * Detects Unicode charset in use utilizing {@link UTFInputStreamReader}. 256 * 257 * @return buffered reader 258 * @throws IOException if any I/O error occurs 259 * @since 9411 260 */ 261 public BufferedReader getContentReader() throws IOException { 262 return new BufferedReader(UTFInputStreamReader.create(getInputStream())); 263 } 264 265 /** 266 * Get local file for the requested resource. 267 * @return The local cache file for URLs. If the resource is a local file, 268 * returns just that file. 269 * @throws IOException when the resource with the given name could not be retrieved 270 */ 271 public synchronized File getFile() throws IOException { 272 if (initialized) 273 return cacheFile; 274 initialized = true; 275 URL url; 276 try { 277 url = new URL(name); 278 if ("file".equals(url.getProtocol())) { 279 cacheFile = new File(name.substring("file:/".length() - 1)); 280 if (!cacheFile.exists()) { 281 cacheFile = new File(name.substring("file://".length() - 1)); 282 } 283 } else { 284 try { 285 cacheFile = checkLocal(url); 286 } catch (SecurityException e) { 287 throw new IOException(e); 288 } 289 } 290 } catch (MalformedURLException e) { 291 if (name == null || name.startsWith("resource://")) { 292 return null; 293 } else if (name.startsWith("josmdir://")) { 294 cacheFile = new File(Config.getDirs().getUserDataDirectory(false), name.substring("josmdir://".length())); 295 } else if (name.startsWith("josmplugindir://")) { 296 cacheFile = new File(Main.pref.getPluginsDirectory(), name.substring("josmplugindir://".length())); 297 } else { 298 cacheFile = new File(name); 299 } 300 } 301 if (cacheFile == null) 302 throw new IOException("Unable to get cache file for "+getName()); 303 return cacheFile; 304 } 305 306 /** 307 * Looks for a certain entry inside a zip file and returns the entry path. 308 * 309 * Replies a file in the top level directory of the ZIP file which has an 310 * extension <code>extension</code>. If more than one files have this 311 * extension, the last file whose name includes <code>namepart</code> 312 * is opened. 313 * 314 * @param extension the extension of the file we're looking for 315 * @param namepart the name part 316 * @return The zip entry path of the matching file. <code>null</code> if this cached file 317 * doesn't represent a zip file or if there was no matching 318 * file in the ZIP file. 319 */ 320 public String findZipEntryPath(String extension, String namepart) { 321 Pair<String, InputStream> ze = findZipEntryImpl(extension, namepart); 322 if (ze == null) return null; 323 return ze.a; 324 } 325 326 /** 327 * Like {@link #findZipEntryPath}, but returns the corresponding InputStream. 328 * @param extension the extension of the file we're looking for 329 * @param namepart the name part 330 * @return InputStream to the matching file. <code>null</code> if this cached file 331 * doesn't represent a zip file or if there was no matching 332 * file in the ZIP file. 333 * @since 6148 334 */ 335 public InputStream findZipEntryInputStream(String extension, String namepart) { 336 Pair<String, InputStream> ze = findZipEntryImpl(extension, namepart); 337 if (ze == null) return null; 338 return ze.b; 339 } 340 341 private Pair<String, InputStream> findZipEntryImpl(String extension, String namepart) { 342 File file = null; 343 try { 344 file = getFile(); 345 } catch (IOException ex) { 346 Logging.log(Logging.LEVEL_WARN, ex); 347 } 348 if (file == null) 349 return null; 350 Pair<String, InputStream> res = null; 351 try { 352 ZipFile zipFile = new ZipFile(file, StandardCharsets.UTF_8); 353 ZipEntry resentry = null; 354 Enumeration<? extends ZipEntry> entries = zipFile.entries(); 355 while (entries.hasMoreElements()) { 356 ZipEntry entry = entries.nextElement(); 357 // choose any file with correct extension. When more than one file, prefer the one which matches namepart 358 if (entry.getName().endsWith('.' + extension) && (resentry == null || entry.getName().indexOf(namepart) >= 0)) { 359 resentry = entry; 360 } 361 } 362 if (resentry != null) { 363 InputStream is = zipFile.getInputStream(resentry); 364 res = Pair.create(resentry.getName(), is); 365 } else { 366 Utils.close(zipFile); 367 } 368 } catch (IOException e) { 369 if (file.getName().endsWith(".zip")) { 370 Logging.log(Logging.LEVEL_WARN, 371 tr("Failed to open file with extension ''{2}'' and namepart ''{3}'' in zip file ''{0}''. Exception was: {1}", 372 file.getName(), e.toString(), extension, namepart), e); 373 } 374 } 375 return res; 376 } 377 378 /** 379 * Clear the cache for the given resource. 380 * This forces a fresh download. 381 * @param name the URL 382 */ 383 public static void cleanup(String name) { 384 cleanup(name, null); 385 } 386 387 /** 388 * Clear the cache for the given resource. 389 * This forces a fresh download. 390 * @param name the URL 391 * @param destDir the destination directory (see {@link #setDestDir(java.lang.String)}) 392 */ 393 public static void cleanup(String name, String destDir) { 394 URL url; 395 try { 396 url = new URL(name); 397 if (!"file".equals(url.getProtocol())) { 398 String prefKey = getPrefKey(url, destDir, null); 399 List<String> localPath = new ArrayList<>(Config.getPref().getList(prefKey)); 400 if (localPath.size() == 2) { 401 File lfile = new File(localPath.get(1)); 402 if (lfile.exists()) { 403 Utils.deleteFile(lfile); 404 } 405 } 406 Config.getPref().putList(prefKey, null); 407 } 408 } catch (MalformedURLException e) { 409 Logging.warn(e); 410 } 411 } 412 413 /** 414 * Get preference key to store the location and age of the cached file. 415 * 2 resources that point to the same url, but that are to be stored in different 416 * directories will not share a cache file. 417 * @param url URL 418 * @param destDir destination directory 419 * @param parameter additional URL parameter (used e.g. for maps) 420 * @return Preference key 421 */ 422 private static String getPrefKey(URL url, String destDir, String parameter) { 423 StringBuilder prefKey = new StringBuilder("mirror."); 424 if (destDir != null) { 425 prefKey.append(destDir).append('.'); 426 } 427 if (parameter != null) { 428 prefKey.append(url.toString().replaceAll("%<(.*)>", "")); 429 } else { 430 prefKey.append(url.toString()); 431 } 432 return prefKey.toString().replaceAll("=", "_"); 433 } 434 435 private File checkLocal(URL url) throws IOException { 436 String prefKey = getPrefKey(url, destDir, parameter); 437 String urlStr = url.toExternalForm(); 438 if (parameter != null) 439 urlStr = urlStr.replaceAll("%<(.*)>", ""); 440 long age = 0L; 441 long maxAgeMillis = maxAge; 442 Long ifModifiedSince = null; 443 File localFile = null; 444 List<String> localPathEntry = new ArrayList<>(Config.getPref().getList(prefKey)); 445 boolean offline = false; 446 try { 447 checkOfflineAccess(urlStr); 448 } catch (OfflineAccessException e) { 449 Logging.trace(e); 450 offline = true; 451 } 452 if (localPathEntry.size() == 2) { 453 localFile = new File(localPathEntry.get(1)); 454 if (!localFile.exists()) { 455 localFile = null; 456 } else { 457 if (maxAge == DEFAULT_MAXTIME 458 || maxAge <= 0 // arbitrary value <= 0 is deprecated 459 ) { 460 maxAgeMillis = TimeUnit.SECONDS.toMillis(Config.getPref().getLong("mirror.maxtime", TimeUnit.DAYS.toSeconds(7))); 461 } 462 age = System.currentTimeMillis() - Long.parseLong(localPathEntry.get(0)); 463 if (offline || age < maxAgeMillis) { 464 return localFile; 465 } 466 if (cachingStrategy == CachingStrategy.IfModifiedSince) { 467 ifModifiedSince = Long.valueOf(localPathEntry.get(0)); 468 } 469 } 470 } 471 if (destDir == null) { 472 destDir = Config.getDirs().getCacheDirectory(true).getPath(); 473 } 474 475 File destDirFile = new File(destDir); 476 if (!destDirFile.exists()) { 477 Utils.mkDirs(destDirFile); 478 } 479 480 // No local file + offline => nothing to do 481 if (offline) { 482 return null; 483 } 484 485 if (parameter != null) { 486 String u = url.toExternalForm(); 487 String uc; 488 if (parameter.isEmpty()) { 489 uc = u.replaceAll("%<(.*)>", ""); 490 } else { 491 uc = u.replaceAll("%<(.*)>", "$1"+parameter); 492 } 493 if (!uc.equals(u)) 494 url = new URL(uc); 495 } 496 497 String a = urlStr.replaceAll("[^A-Za-z0-9_.-]", "_"); 498 String localPath = "mirror_" + a; 499 destDirFile = new File(destDir, localPath + ".tmp"); 500 try { 501 activeConnection = HttpClient.create(url) 502 .setAccept(httpAccept) 503 .setIfModifiedSince(ifModifiedSince == null ? 0L : ifModifiedSince) 504 .setHeaders(httpHeaders); 505 if (fastFail) { 506 activeConnection.setReadTimeout(1000); 507 } 508 final HttpClient.Response con = activeConnection.connect(); 509 if (ifModifiedSince != null && con.getResponseCode() == HttpURLConnection.HTTP_NOT_MODIFIED) { 510 Logging.debug("304 Not Modified ({0})", urlStr); 511 if (localFile == null) 512 throw new AssertionError(); 513 Config.getPref().putList(prefKey, 514 Arrays.asList(Long.toString(System.currentTimeMillis()), localPathEntry.get(1))); 515 return localFile; 516 } else if (con.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) { 517 throw new IOException(tr("The requested URL {0} was not found", urlStr)); 518 } 519 try (InputStream is = con.getContent()) { 520 Files.copy(is, destDirFile.toPath(), StandardCopyOption.REPLACE_EXISTING); 521 } 522 activeConnection = null; 523 localFile = new File(destDir, localPath); 524 if (Main.platform.rename(destDirFile, localFile)) { 525 Config.getPref().putList(prefKey, 526 Arrays.asList(Long.toString(System.currentTimeMillis()), localFile.toString())); 527 } else { 528 Logging.warn(tr("Failed to rename file {0} to {1}.", 529 destDirFile.getPath(), localFile.getPath())); 530 } 531 } catch (IOException e) { 532 if (age >= maxAgeMillis && age < maxAgeMillis*2) { 533 Logging.warn(tr("Failed to load {0}, use cached file and retry next time: {1}", urlStr, e)); 534 return localFile; 535 } else { 536 throw e; 537 } 538 } 539 540 return localFile; 541 } 542 543 private static void checkOfflineAccess(String urlString) { 544 OnlineResource.JOSM_WEBSITE.checkOfflineAccess(urlString, Main.getJOSMWebsite()); 545 OnlineResource.OSM_API.checkOfflineAccess(urlString, OsmApi.getOsmApi().getServerUrl()); 546 } 547 548 /** 549 * Attempts to disconnect an URL connection. 550 * @see HttpClient#disconnect() 551 * @since 9411 552 */ 553 @Override 554 public void close() { 555 if (activeConnection != null) { 556 activeConnection.disconnect(); 557 } 558 } 559 560 /** 561 * Clears the cached file 562 * @throws IOException if any I/O error occurs 563 * @since 10993 564 */ 565 public void clear() throws IOException { 566 URL url; 567 try { 568 url = new URL(name); 569 if ("file".equals(url.getProtocol())) { 570 return; // this is local file - do not delete it 571 } 572 } catch (MalformedURLException e) { 573 return; // if it's not a URL, then it still might be a local file - better not to delete 574 } 575 File f = getFile(); 576 if (f != null && f.exists()) { 577 Utils.deleteFile(f); 578 } 579 } 580}