001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.data.imagery; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.io.ByteArrayInputStream; 007import java.io.IOException; 008import java.net.URL; 009import java.nio.charset.StandardCharsets; 010import java.util.HashSet; 011import java.util.List; 012import java.util.Map; 013import java.util.Map.Entry; 014import java.util.Optional; 015import java.util.Set; 016import java.util.concurrent.ConcurrentHashMap; 017import java.util.concurrent.ConcurrentMap; 018import java.util.concurrent.ThreadPoolExecutor; 019import java.util.concurrent.TimeUnit; 020import java.util.regex.Matcher; 021import java.util.regex.Pattern; 022 023import org.apache.commons.jcs.access.behavior.ICacheAccess; 024import org.openstreetmap.gui.jmapviewer.Tile; 025import org.openstreetmap.gui.jmapviewer.interfaces.TileJob; 026import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener; 027import org.openstreetmap.gui.jmapviewer.interfaces.TileSource; 028import org.openstreetmap.gui.jmapviewer.tilesources.AbstractTMSTileSource; 029import org.openstreetmap.josm.data.cache.BufferedImageCacheEntry; 030import org.openstreetmap.josm.data.cache.CacheEntry; 031import org.openstreetmap.josm.data.cache.CacheEntryAttributes; 032import org.openstreetmap.josm.data.cache.ICachedLoaderListener; 033import org.openstreetmap.josm.data.cache.JCSCachedTileLoaderJob; 034import org.openstreetmap.josm.data.preferences.LongProperty; 035import org.openstreetmap.josm.tools.HttpClient; 036import org.openstreetmap.josm.tools.Logging; 037import org.openstreetmap.josm.tools.Utils; 038 039/** 040 * Class bridging TMS requests to JCS cache requests 041 * 042 * @author Wiktor Niesiobędzki 043 * @since 8168 044 */ 045public class TMSCachedTileLoaderJob extends JCSCachedTileLoaderJob<String, BufferedImageCacheEntry> implements TileJob, ICachedLoaderListener { 046 /** General maximum expires for tiles. Might be overridden by imagery settings */ 047 public static final LongProperty MAXIMUM_EXPIRES = new LongProperty("imagery.generic.maximum_expires", TimeUnit.DAYS.toMillis(30)); 048 /** General minimum expires for tiles. Might be overridden by imagery settings */ 049 public static final LongProperty MINIMUM_EXPIRES = new LongProperty("imagery.generic.minimum_expires", TimeUnit.HOURS.toMillis(1)); 050 static final Pattern SERVICE_EXCEPTION_PATTERN = Pattern.compile("(?s).+<ServiceException[^>]*>(.+)</ServiceException>.+"); 051 protected final Tile tile; 052 private volatile URL url; 053 private final TileJobOptions options; 054 055 // we need another deduplication of Tile Loader listeners, as for each submit, new TMSCachedTileLoaderJob was created 056 // that way, we reduce calls to tileLoadingFinished, and general CPU load due to surplus Map repaints 057 private static final ConcurrentMap<String, Set<TileLoaderListener>> inProgress = new ConcurrentHashMap<>(); 058 059 /** 060 * Constructor for creating a job, to get a specific tile from cache 061 * @param listener Tile loader listener 062 * @param tile to be fetched from cache 063 * @param cache object 064 * @param options for job (such as http headers, timeouts etc.) 065 * @param downloadExecutor that will be executing the jobs 066 */ 067 068 public TMSCachedTileLoaderJob(TileLoaderListener listener, Tile tile, 069 ICacheAccess<String, BufferedImageCacheEntry> cache, 070 TileJobOptions options, 071 ThreadPoolExecutor downloadExecutor) { 072 super(cache, options, downloadExecutor); 073 this.tile = tile; 074 this.options = options; 075 if (listener != null) { 076 String deduplicationKey = getCacheKey(); 077 synchronized (inProgress) { 078 inProgress.computeIfAbsent(deduplicationKey, k -> new HashSet<>()).add(listener); 079 } 080 } 081 } 082 083 @Override 084 public String getCacheKey() { 085 if (tile != null) { 086 TileSource tileSource = tile.getTileSource(); 087 return Optional.ofNullable(tileSource.getName()).orElse("").replace(':', '_') + ':' 088 + tileSource.getTileId(tile.getZoom(), tile.getXtile(), tile.getYtile()); 089 } 090 return null; 091 } 092 093 /* 094 * this doesn't needs to be synchronized, as it's not that costly to keep only one execution 095 * in parallel, but URL creation and Tile.getUrl() are costly and are not needed when fetching 096 * data from cache, that's why URL creation is postponed until it's needed 097 * 098 * We need to have static url value for TileLoaderJob, as for some TileSources we might get different 099 * URL's each call we made (servers switching), and URL's are used below as a key for duplicate detection 100 * 101 */ 102 @Override 103 public URL getUrl() throws IOException { 104 if (url == null) { 105 synchronized (this) { 106 if (url == null) { 107 String sUrl = tile.getUrl(); 108 if (!"".equals(sUrl)) { 109 url = new URL(sUrl); 110 } 111 } 112 } 113 } 114 return url; 115 } 116 117 @Override 118 public boolean isObjectLoadable() { 119 if (cacheData != null) { 120 byte[] content = cacheData.getContent(); 121 try { 122 return content.length > 0 || cacheData.getImage() != null || isNoTileAtZoom(); 123 } catch (IOException e) { 124 Logging.logWithStackTrace(Logging.LEVEL_WARN, e, "JCS TMS - error loading from cache for tile {0}: {1}", 125 tile.getKey(), e.getMessage()); 126 } 127 } 128 return false; 129 } 130 131 @Override 132 protected boolean isResponseLoadable(Map<String, List<String>> headers, int statusCode, byte[] content) { 133 attributes.setMetadata(tile.getTileSource().getMetadata(headers)); 134 if (tile.getTileSource().isNoTileAtZoom(headers, statusCode, content)) { 135 attributes.setNoTileAtZoom(true); 136 return false; // do no try to load data from no-tile at zoom, cache empty object instead 137 } 138 return super.isResponseLoadable(headers, statusCode, content); 139 } 140 141 @Override 142 protected boolean cacheAsEmpty() { 143 return isNoTileAtZoom() || super.cacheAsEmpty(); 144 } 145 146 @Override 147 public void submit(boolean force) { 148 tile.initLoading(); 149 try { 150 super.submit(this, force); 151 } catch (IOException | IllegalArgumentException e) { 152 // if we fail to submit the job, mark tile as loaded and set error message 153 Logging.log(Logging.LEVEL_WARN, e); 154 tile.finishLoading(); 155 tile.setError(e.getMessage()); 156 } 157 } 158 159 @Override 160 public void loadingFinished(CacheEntry object, CacheEntryAttributes attributes, LoadResult result) { 161 this.attributes = attributes; // as we might get notification from other object than our selfs, pass attributes along 162 Set<TileLoaderListener> listeners; 163 synchronized (inProgress) { 164 listeners = inProgress.remove(getCacheKey()); 165 } 166 boolean status = result.equals(LoadResult.SUCCESS); 167 168 try { 169 tile.finishLoading(); // whatever happened set that loading has finished 170 // set tile metadata 171 if (this.attributes != null) { 172 for (Entry<String, String> e: this.attributes.getMetadata().entrySet()) { 173 tile.putValue(e.getKey(), e.getValue()); 174 } 175 } 176 177 switch(result) { 178 case SUCCESS: 179 handleNoTileAtZoom(); 180 if (attributes != null) { 181 int httpStatusCode = attributes.getResponseCode(); 182 if (httpStatusCode >= 400 && !isNoTileAtZoom()) { 183 if (attributes.getErrorMessage() == null) { 184 tile.setError(tr("HTTP error {0} when loading tiles", httpStatusCode)); 185 } else { 186 tile.setError(tr("Error downloading tiles: {0}", attributes.getErrorMessage())); 187 } 188 status = false; 189 } 190 } 191 status &= tryLoadTileImage(object); //try to keep returned image as background 192 break; 193 case FAILURE: 194 tile.setError("Problem loading tile"); 195 tryLoadTileImage(object); 196 break; 197 case CANCELED: 198 tile.loadingCanceled(); 199 // do nothing 200 } 201 202 // always check, if there is some listener interested in fact, that tile has finished loading 203 if (listeners != null) { // listeners might be null, if some other thread notified already about success 204 for (TileLoaderListener l: listeners) { 205 l.tileLoadingFinished(tile, status); 206 } 207 } 208 } catch (IOException e) { 209 Logging.warn("JCS TMS - error loading object for tile {0}: {1}", tile.getKey(), e.getMessage()); 210 tile.setError(e); 211 tile.setLoaded(false); 212 if (listeners != null) { // listeners might be null, if some other thread notified already about success 213 for (TileLoaderListener l: listeners) { 214 l.tileLoadingFinished(tile, false); 215 } 216 } 217 } 218 } 219 220 /** 221 * For TMS use BaseURL as settings discovery, so for different paths, we will have different settings (useful for developer servers) 222 * 223 * @return base URL of TMS or server url as defined in super class 224 */ 225 @Override 226 protected String getServerKey() { 227 TileSource ts = tile.getSource(); 228 if (ts instanceof AbstractTMSTileSource) { 229 return ((AbstractTMSTileSource) ts).getBaseUrl(); 230 } 231 return super.getServerKey(); 232 } 233 234 @Override 235 protected BufferedImageCacheEntry createCacheEntry(byte[] content) { 236 return new BufferedImageCacheEntry(content); 237 } 238 239 @Override 240 public void submit() { 241 submit(false); 242 } 243 244 @Override 245 protected CacheEntryAttributes parseHeaders(HttpClient.Response urlConn) { 246 CacheEntryAttributes ret = super.parseHeaders(urlConn); 247 // keep the expiration time between MINIMUM_EXPIRES and MAXIMUM_EXPIRES, so we will cache the tiles 248 // at least for some short period of time, but not too long 249 if (ret.getExpirationTime() < now + Math.max(MINIMUM_EXPIRES.get(), options.getMinimumExpiryTime())) { 250 ret.setExpirationTime(now + Math.max(MINIMUM_EXPIRES.get(), TimeUnit.SECONDS.toMillis(options.getMinimumExpiryTime()))); 251 } 252 if (ret.getExpirationTime() > now + Math.max(MAXIMUM_EXPIRES.get(), options.getMinimumExpiryTime())) { 253 ret.setExpirationTime(now + Math.max(MAXIMUM_EXPIRES.get(), TimeUnit.SECONDS.toMillis(options.getMinimumExpiryTime()))); 254 } 255 return ret; 256 } 257 258 private boolean handleNoTileAtZoom() { 259 if (isNoTileAtZoom()) { 260 Logging.debug("JCS TMS - Tile valid, but no file, as no tiles at this level {0}", tile); 261 tile.setError("No tile at this zoom level"); 262 tile.putValue("tile-info", "no-tile"); 263 return true; 264 } 265 return false; 266 } 267 268 private boolean isNoTileAtZoom() { 269 if (attributes == null) { 270 Logging.warn("Cache attributes are null"); 271 } 272 return attributes != null && attributes.isNoTileAtZoom(); 273 } 274 275 private boolean tryLoadTileImage(CacheEntry object) throws IOException { 276 if (object != null) { 277 byte[] content = object.getContent(); 278 if (content.length > 0) { 279 try (ByteArrayInputStream in = new ByteArrayInputStream(content)) { 280 tile.loadImage(in); 281 if (tile.getImage() == null) { 282 String s = new String(content, StandardCharsets.UTF_8); 283 Matcher m = SERVICE_EXCEPTION_PATTERN.matcher(s); 284 if (m.matches()) { 285 String message = Utils.strip(m.group(1)); 286 tile.setError(message); 287 Logging.error(message); 288 Logging.debug(s); 289 } else { 290 tile.setError(tr("Could not load image from tile server")); 291 } 292 return false; 293 } 294 } catch (UnsatisfiedLinkError | SecurityException e) { 295 throw new IOException(e); 296 } 297 } 298 } 299 return true; 300 } 301}