001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.data.imagery; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.io.ByteArrayInputStream; 007import java.io.IOException; 008import java.net.URL; 009import java.nio.charset.StandardCharsets; 010import java.util.HashSet; 011import java.util.List; 012import java.util.Map; 013import java.util.Map.Entry; 014import java.util.Optional; 015import java.util.Set; 016import java.util.concurrent.ConcurrentHashMap; 017import java.util.concurrent.ConcurrentMap; 018import java.util.concurrent.ThreadPoolExecutor; 019import java.util.concurrent.TimeUnit; 020import java.util.regex.Matcher; 021import java.util.regex.Pattern; 022 023import org.apache.commons.jcs.access.behavior.ICacheAccess; 024import org.openstreetmap.gui.jmapviewer.Tile; 025import org.openstreetmap.gui.jmapviewer.interfaces.TileJob; 026import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener; 027import org.openstreetmap.gui.jmapviewer.interfaces.TileSource; 028import org.openstreetmap.gui.jmapviewer.tilesources.AbstractTMSTileSource; 029import org.openstreetmap.josm.data.cache.BufferedImageCacheEntry; 030import org.openstreetmap.josm.data.cache.CacheEntry; 031import org.openstreetmap.josm.data.cache.CacheEntryAttributes; 032import org.openstreetmap.josm.data.cache.ICachedLoaderListener; 033import org.openstreetmap.josm.data.cache.JCSCachedTileLoaderJob; 034import org.openstreetmap.josm.data.preferences.LongProperty; 035import org.openstreetmap.josm.tools.HttpClient; 036import org.openstreetmap.josm.tools.Logging; 037import org.openstreetmap.josm.tools.Utils; 038 039/** 040 * Class bridging TMS requests to JCS cache requests 041 * 042 * @author Wiktor Niesiobędzki 043 * @since 8168 044 */ 045public class TMSCachedTileLoaderJob extends JCSCachedTileLoaderJob<String, BufferedImageCacheEntry> implements TileJob, ICachedLoaderListener { 046 private static final LongProperty MAXIMUM_EXPIRES = new LongProperty("imagery.generic.maximum_expires", TimeUnit.DAYS.toMillis(30)); 047 private static final LongProperty MINIMUM_EXPIRES = new LongProperty("imagery.generic.minimum_expires", TimeUnit.HOURS.toMillis(1)); 048 static final Pattern SERVICE_EXCEPTION_PATTERN = Pattern.compile("(?s).+<ServiceException[^>]*>(.+)</ServiceException>.+"); 049 protected final Tile tile; 050 private volatile URL url; 051 052 // we need another deduplication of Tile Loader listeners, as for each submit, new TMSCachedTileLoaderJob was created 053 // that way, we reduce calls to tileLoadingFinished, and general CPU load due to surplus Map repaints 054 private static final ConcurrentMap<String, Set<TileLoaderListener>> inProgress = new ConcurrentHashMap<>(); 055 056 /** 057 * Constructor for creating a job, to get a specific tile from cache 058 * @param listener Tile loader listener 059 * @param tile to be fetched from cache 060 * @param cache object 061 * @param connectTimeout when connecting to remote resource 062 * @param readTimeout when connecting to remote resource 063 * @param headers HTTP headers to be sent together with request 064 * @param downloadExecutor that will be executing the jobs 065 */ 066 public TMSCachedTileLoaderJob(TileLoaderListener listener, Tile tile, 067 ICacheAccess<String, BufferedImageCacheEntry> cache, 068 int connectTimeout, int readTimeout, Map<String, String> headers, 069 ThreadPoolExecutor downloadExecutor) { 070 super(cache, connectTimeout, readTimeout, headers, downloadExecutor); 071 this.tile = tile; 072 if (listener != null) { 073 String deduplicationKey = getCacheKey(); 074 synchronized (inProgress) { 075 inProgress.computeIfAbsent(deduplicationKey, k -> new HashSet<>()).add(listener); 076 } 077 } 078 } 079 080 @Override 081 public String getCacheKey() { 082 if (tile != null) { 083 TileSource tileSource = tile.getTileSource(); 084 return Optional.ofNullable(tileSource.getName()).orElse("").replace(':', '_') + ':' 085 + tileSource.getTileId(tile.getZoom(), tile.getXtile(), tile.getYtile()); 086 } 087 return null; 088 } 089 090 /* 091 * this doesn't needs to be synchronized, as it's not that costly to keep only one execution 092 * in parallel, but URL creation and Tile.getUrl() are costly and are not needed when fetching 093 * data from cache, that's why URL creation is postponed until it's needed 094 * 095 * We need to have static url value for TileLoaderJob, as for some TileSources we might get different 096 * URL's each call we made (servers switching), and URL's are used below as a key for duplicate detection 097 * 098 */ 099 @Override 100 public URL getUrl() throws IOException { 101 if (url == null) { 102 synchronized (this) { 103 if (url == null) { 104 String sUrl = tile.getUrl(); 105 if (!"".equals(sUrl)) { 106 url = new URL(sUrl); 107 } 108 } 109 } 110 } 111 return url; 112 } 113 114 @Override 115 public boolean isObjectLoadable() { 116 if (cacheData != null) { 117 byte[] content = cacheData.getContent(); 118 try { 119 return content.length > 0 || cacheData.getImage() != null || isNoTileAtZoom(); 120 } catch (IOException e) { 121 Logging.logWithStackTrace(Logging.LEVEL_WARN, e, "JCS TMS - error loading from cache for tile {0}: {1}", 122 new Object[] {tile.getKey(), e.getMessage()} 123 ); 124 } 125 } 126 return false; 127 } 128 129 @Override 130 protected boolean isResponseLoadable(Map<String, List<String>> headers, int statusCode, byte[] content) { 131 attributes.setMetadata(tile.getTileSource().getMetadata(headers)); 132 if (tile.getTileSource().isNoTileAtZoom(headers, statusCode, content)) { 133 attributes.setNoTileAtZoom(true); 134 return false; // do no try to load data from no-tile at zoom, cache empty object instead 135 } 136 return super.isResponseLoadable(headers, statusCode, content); 137 } 138 139 @Override 140 protected boolean cacheAsEmpty() { 141 return isNoTileAtZoom() || super.cacheAsEmpty(); 142 } 143 144 @Override 145 public void submit(boolean force) { 146 tile.initLoading(); 147 try { 148 super.submit(this, force); 149 } catch (IOException | IllegalArgumentException e) { 150 // if we fail to submit the job, mark tile as loaded and set error message 151 Logging.log(Logging.LEVEL_WARN, e); 152 tile.finishLoading(); 153 tile.setError(e.getMessage()); 154 } 155 } 156 157 @Override 158 public void loadingFinished(CacheEntry object, CacheEntryAttributes attributes, LoadResult result) { 159 this.attributes = attributes; // as we might get notification from other object than our selfs, pass attributes along 160 Set<TileLoaderListener> listeners; 161 synchronized (inProgress) { 162 listeners = inProgress.remove(getCacheKey()); 163 } 164 boolean status = result.equals(LoadResult.SUCCESS); 165 166 try { 167 tile.finishLoading(); // whatever happened set that loading has finished 168 // set tile metadata 169 if (this.attributes != null) { 170 for (Entry<String, String> e: this.attributes.getMetadata().entrySet()) { 171 tile.putValue(e.getKey(), e.getValue()); 172 } 173 } 174 175 switch(result) { 176 case SUCCESS: 177 handleNoTileAtZoom(); 178 if (attributes != null) { 179 int httpStatusCode = attributes.getResponseCode(); 180 if (httpStatusCode >= 400 && !isNoTileAtZoom()) { 181 if (attributes.getErrorMessage() == null) { 182 tile.setError(tr("HTTP error {0} when loading tiles", httpStatusCode)); 183 } else { 184 tile.setError(tr("Error downloading tiles: {0}", attributes.getErrorMessage())); 185 } 186 status = false; 187 } 188 } 189 status &= tryLoadTileImage(object); //try to keep returned image as background 190 break; 191 case FAILURE: 192 tile.setError("Problem loading tile"); 193 tryLoadTileImage(object); 194 break; 195 case CANCELED: 196 tile.loadingCanceled(); 197 // do nothing 198 } 199 200 // always check, if there is some listener interested in fact, that tile has finished loading 201 if (listeners != null) { // listeners might be null, if some other thread notified already about success 202 for (TileLoaderListener l: listeners) { 203 l.tileLoadingFinished(tile, status); 204 } 205 } 206 } catch (IOException e) { 207 Logging.warn("JCS TMS - error loading object for tile {0}: {1}", new Object[] {tile.getKey(), e.getMessage()}); 208 tile.setError(e); 209 tile.setLoaded(false); 210 if (listeners != null) { // listeners might be null, if some other thread notified already about success 211 for (TileLoaderListener l: listeners) { 212 l.tileLoadingFinished(tile, false); 213 } 214 } 215 } 216 } 217 218 /** 219 * For TMS use BaseURL as settings discovery, so for different paths, we will have different settings (useful for developer servers) 220 * 221 * @return base URL of TMS or server url as defined in super class 222 */ 223 @Override 224 protected String getServerKey() { 225 TileSource ts = tile.getSource(); 226 if (ts instanceof AbstractTMSTileSource) { 227 return ((AbstractTMSTileSource) ts).getBaseUrl(); 228 } 229 return super.getServerKey(); 230 } 231 232 @Override 233 protected BufferedImageCacheEntry createCacheEntry(byte[] content) { 234 return new BufferedImageCacheEntry(content); 235 } 236 237 @Override 238 public void submit() { 239 submit(false); 240 } 241 242 @Override 243 protected CacheEntryAttributes parseHeaders(HttpClient.Response urlConn) { 244 CacheEntryAttributes ret = super.parseHeaders(urlConn); 245 // keep the expiration time between MINIMUM_EXPIRES and MAXIMUM_EXPIRES, so we will cache the tiles 246 // at least for some short period of time, but not too long 247 if (ret.getExpirationTime() < now + MINIMUM_EXPIRES.get()) { 248 ret.setExpirationTime(now + MINIMUM_EXPIRES.get()); 249 } 250 if (ret.getExpirationTime() > now + MAXIMUM_EXPIRES.get()) { 251 ret.setExpirationTime(now + MAXIMUM_EXPIRES.get()); 252 } 253 return ret; 254 } 255 256 private boolean handleNoTileAtZoom() { 257 if (isNoTileAtZoom()) { 258 Logging.debug("JCS TMS - Tile valid, but no file, as no tiles at this level {0}", tile); 259 tile.setError("No tile at this zoom level"); 260 tile.putValue("tile-info", "no-tile"); 261 return true; 262 } 263 return false; 264 } 265 266 private boolean isNoTileAtZoom() { 267 if (attributes == null) { 268 Logging.warn("Cache attributes are null"); 269 } 270 return attributes != null && attributes.isNoTileAtZoom(); 271 } 272 273 private boolean tryLoadTileImage(CacheEntry object) throws IOException { 274 if (object != null) { 275 byte[] content = object.getContent(); 276 if (content.length > 0) { 277 try (ByteArrayInputStream in = new ByteArrayInputStream(content)) { 278 tile.loadImage(in); 279 if (tile.getImage() == null) { 280 String s = new String(content, StandardCharsets.UTF_8); 281 Matcher m = SERVICE_EXCEPTION_PATTERN.matcher(s); 282 if (m.matches()) { 283 String message = Utils.strip(m.group(1)); 284 tile.setError(message); 285 Logging.error(message); 286 Logging.debug(s); 287 } else { 288 tile.setError(tr("Could not load image from tile server")); 289 } 290 return false; 291 } 292 } catch (UnsatisfiedLinkError e) { 293 throw new IOException(e); 294 } 295 } 296 } 297 return true; 298 } 299}