001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.data.imagery;
003
004import static org.openstreetmap.josm.tools.I18n.tr;
005
006import java.io.ByteArrayInputStream;
007import java.io.IOException;
008import java.net.URL;
009import java.nio.charset.StandardCharsets;
010import java.util.HashSet;
011import java.util.List;
012import java.util.Map;
013import java.util.Map.Entry;
014import java.util.Optional;
015import java.util.Set;
016import java.util.concurrent.ConcurrentHashMap;
017import java.util.concurrent.ConcurrentMap;
018import java.util.concurrent.ThreadPoolExecutor;
019import java.util.concurrent.TimeUnit;
020import java.util.regex.Matcher;
021import java.util.regex.Pattern;
022
023import org.apache.commons.jcs.access.behavior.ICacheAccess;
024import org.openstreetmap.gui.jmapviewer.Tile;
025import org.openstreetmap.gui.jmapviewer.interfaces.TileJob;
026import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener;
027import org.openstreetmap.gui.jmapviewer.interfaces.TileSource;
028import org.openstreetmap.gui.jmapviewer.tilesources.AbstractTMSTileSource;
029import org.openstreetmap.josm.data.cache.BufferedImageCacheEntry;
030import org.openstreetmap.josm.data.cache.CacheEntry;
031import org.openstreetmap.josm.data.cache.CacheEntryAttributes;
032import org.openstreetmap.josm.data.cache.ICachedLoaderListener;
033import org.openstreetmap.josm.data.cache.JCSCachedTileLoaderJob;
034import org.openstreetmap.josm.data.preferences.LongProperty;
035import org.openstreetmap.josm.tools.HttpClient;
036import org.openstreetmap.josm.tools.Logging;
037import org.openstreetmap.josm.tools.Utils;
038
039/**
040 * Class bridging TMS requests to JCS cache requests
041 *
042 * @author Wiktor Niesiobędzki
043 * @since 8168
044 */
045public class TMSCachedTileLoaderJob extends JCSCachedTileLoaderJob<String, BufferedImageCacheEntry> implements TileJob, ICachedLoaderListener {
046    /** General maximum expires for tiles. Might be overridden by imagery settings */
047    public static final LongProperty MAXIMUM_EXPIRES = new LongProperty("imagery.generic.maximum_expires", TimeUnit.DAYS.toMillis(30));
048    /** General minimum expires for tiles. Might be overridden by imagery settings */
049    public static final LongProperty MINIMUM_EXPIRES = new LongProperty("imagery.generic.minimum_expires", TimeUnit.HOURS.toMillis(1));
050    static final Pattern SERVICE_EXCEPTION_PATTERN = Pattern.compile("(?s).+<ServiceException[^>]*>(.+)</ServiceException>.+");
051    protected final Tile tile;
052    private volatile URL url;
053    private final TileJobOptions options;
054
055    // we need another deduplication of Tile Loader listeners, as for each submit, new TMSCachedTileLoaderJob was created
056    // that way, we reduce calls to tileLoadingFinished, and general CPU load due to surplus Map repaints
057    private static final ConcurrentMap<String, Set<TileLoaderListener>> inProgress = new ConcurrentHashMap<>();
058
059    /**
060     * Constructor for creating a job, to get a specific tile from cache
061     * @param listener Tile loader listener
062     * @param tile to be fetched from cache
063     * @param cache object
064     * @param options for job (such as http headers, timeouts etc.)
065     * @param downloadExecutor that will be executing the jobs
066     */
067
068    public TMSCachedTileLoaderJob(TileLoaderListener listener, Tile tile,
069            ICacheAccess<String, BufferedImageCacheEntry> cache,
070            TileJobOptions options,
071            ThreadPoolExecutor downloadExecutor) {
072        super(cache, options, downloadExecutor);
073        this.tile = tile;
074        this.options = options;
075        if (listener != null) {
076            String deduplicationKey = getCacheKey();
077            synchronized (inProgress) {
078                inProgress.computeIfAbsent(deduplicationKey, k -> new HashSet<>()).add(listener);
079            }
080        }
081    }
082
083    @Override
084    public String getCacheKey() {
085        if (tile != null) {
086            TileSource tileSource = tile.getTileSource();
087            return Optional.ofNullable(tileSource.getName()).orElse("").replace(':', '_') + ':'
088                    + tileSource.getTileId(tile.getZoom(), tile.getXtile(), tile.getYtile());
089        }
090        return null;
091    }
092
093    /*
094     *  this doesn't needs to be synchronized, as it's not that costly to keep only one execution
095     *  in parallel, but URL creation and Tile.getUrl() are costly and are not needed when fetching
096     *  data from cache, that's why URL creation is postponed until it's needed
097     *
098     *  We need to have static url value for TileLoaderJob, as for some TileSources we might get different
099     *  URL's each call we made (servers switching), and URL's are used below as a key for duplicate detection
100     *
101     */
102    @Override
103    public URL getUrl() throws IOException {
104        if (url == null) {
105            synchronized (this) {
106                if (url == null) {
107                    String sUrl = tile.getUrl();
108                    if (!"".equals(sUrl)) {
109                        url = new URL(sUrl);
110                    }
111                }
112            }
113        }
114        return url;
115    }
116
117    @Override
118    public boolean isObjectLoadable() {
119        if (cacheData != null) {
120            byte[] content = cacheData.getContent();
121            try {
122                return content.length > 0 || cacheData.getImage() != null || isNoTileAtZoom();
123            } catch (IOException e) {
124                Logging.logWithStackTrace(Logging.LEVEL_WARN, e, "JCS TMS - error loading from cache for tile {0}: {1}",
125                        tile.getKey(), e.getMessage());
126            }
127        }
128        return false;
129    }
130
131    @Override
132    protected boolean isResponseLoadable(Map<String, List<String>> headers, int statusCode, byte[] content) {
133        attributes.setMetadata(tile.getTileSource().getMetadata(headers));
134        if (tile.getTileSource().isNoTileAtZoom(headers, statusCode, content)) {
135            attributes.setNoTileAtZoom(true);
136            return false; // do no try to load data from no-tile at zoom, cache empty object instead
137        }
138        return super.isResponseLoadable(headers, statusCode, content);
139    }
140
141    @Override
142    protected boolean cacheAsEmpty() {
143        return isNoTileAtZoom() || super.cacheAsEmpty();
144    }
145
146    @Override
147    public void submit(boolean force) {
148        tile.initLoading();
149        try {
150            super.submit(this, force);
151        } catch (IOException | IllegalArgumentException e) {
152            // if we fail to submit the job, mark tile as loaded and set error message
153            Logging.log(Logging.LEVEL_WARN, e);
154            tile.finishLoading();
155            tile.setError(e.getMessage());
156        }
157    }
158
159    @Override
160    public void loadingFinished(CacheEntry object, CacheEntryAttributes attributes, LoadResult result) {
161        this.attributes = attributes; // as we might get notification from other object than our selfs, pass attributes along
162        Set<TileLoaderListener> listeners;
163        synchronized (inProgress) {
164            listeners = inProgress.remove(getCacheKey());
165        }
166        boolean status = result.equals(LoadResult.SUCCESS);
167
168        try {
169            tile.finishLoading(); // whatever happened set that loading has finished
170            // set tile metadata
171            if (this.attributes != null) {
172                for (Entry<String, String> e: this.attributes.getMetadata().entrySet()) {
173                    tile.putValue(e.getKey(), e.getValue());
174                }
175            }
176
177            switch(result) {
178            case SUCCESS:
179                handleNoTileAtZoom();
180                if (attributes != null) {
181                    int httpStatusCode = attributes.getResponseCode();
182                    if (httpStatusCode >= 400 && !isNoTileAtZoom()) {
183                        if (attributes.getErrorMessage() == null) {
184                            tile.setError(tr("HTTP error {0} when loading tiles", httpStatusCode));
185                        } else {
186                            tile.setError(tr("Error downloading tiles: {0}", attributes.getErrorMessage()));
187                        }
188                        status = false;
189                    }
190                }
191                status &= tryLoadTileImage(object); //try to keep returned image as background
192                break;
193            case FAILURE:
194                tile.setError("Problem loading tile");
195                tryLoadTileImage(object);
196                break;
197            case CANCELED:
198                tile.loadingCanceled();
199                // do nothing
200            }
201
202            // always check, if there is some listener interested in fact, that tile has finished loading
203            if (listeners != null) { // listeners might be null, if some other thread notified already about success
204                for (TileLoaderListener l: listeners) {
205                    l.tileLoadingFinished(tile, status);
206                }
207            }
208        } catch (IOException e) {
209            Logging.warn("JCS TMS - error loading object for tile {0}: {1}", tile.getKey(), e.getMessage());
210            tile.setError(e);
211            tile.setLoaded(false);
212            if (listeners != null) { // listeners might be null, if some other thread notified already about success
213                for (TileLoaderListener l: listeners) {
214                    l.tileLoadingFinished(tile, false);
215                }
216            }
217        }
218    }
219
220    /**
221     * For TMS use BaseURL as settings discovery, so for different paths, we will have different settings (useful for developer servers)
222     *
223     * @return base URL of TMS or server url as defined in super class
224     */
225    @Override
226    protected String getServerKey() {
227        TileSource ts = tile.getSource();
228        if (ts instanceof AbstractTMSTileSource) {
229            return ((AbstractTMSTileSource) ts).getBaseUrl();
230        }
231        return super.getServerKey();
232    }
233
234    @Override
235    protected BufferedImageCacheEntry createCacheEntry(byte[] content) {
236        return new BufferedImageCacheEntry(content);
237    }
238
239    @Override
240    public void submit() {
241        submit(false);
242    }
243
244    @Override
245    protected CacheEntryAttributes parseHeaders(HttpClient.Response urlConn) {
246        CacheEntryAttributes ret = super.parseHeaders(urlConn);
247        // keep the expiration time between MINIMUM_EXPIRES and MAXIMUM_EXPIRES, so we will cache the tiles
248        // at least for some short period of time, but not too long
249        if (ret.getExpirationTime() < now + Math.max(MINIMUM_EXPIRES.get(), options.getMinimumExpiryTime())) {
250            ret.setExpirationTime(now + Math.max(MINIMUM_EXPIRES.get(), TimeUnit.SECONDS.toMillis(options.getMinimumExpiryTime())));
251        }
252        if (ret.getExpirationTime() > now + Math.max(MAXIMUM_EXPIRES.get(), options.getMinimumExpiryTime())) {
253            ret.setExpirationTime(now + Math.max(MAXIMUM_EXPIRES.get(), TimeUnit.SECONDS.toMillis(options.getMinimumExpiryTime())));
254        }
255        return ret;
256    }
257
258    private boolean handleNoTileAtZoom() {
259        if (isNoTileAtZoom()) {
260            Logging.debug("JCS TMS - Tile valid, but no file, as no tiles at this level {0}", tile);
261            tile.setError("No tile at this zoom level");
262            tile.putValue("tile-info", "no-tile");
263            return true;
264        }
265        return false;
266    }
267
268    private boolean isNoTileAtZoom() {
269        if (attributes == null) {
270            Logging.warn("Cache attributes are null");
271        }
272        return attributes != null && attributes.isNoTileAtZoom();
273    }
274
275    private boolean tryLoadTileImage(CacheEntry object) throws IOException {
276        if (object != null) {
277            byte[] content = object.getContent();
278            if (content.length > 0) {
279                try (ByteArrayInputStream in = new ByteArrayInputStream(content)) {
280                    tile.loadImage(in);
281                    if (tile.getImage() == null) {
282                        String s = new String(content, StandardCharsets.UTF_8);
283                        Matcher m = SERVICE_EXCEPTION_PATTERN.matcher(s);
284                        if (m.matches()) {
285                            String message = Utils.strip(m.group(1));
286                            tile.setError(message);
287                            Logging.error(message);
288                            Logging.debug(s);
289                        } else {
290                            tile.setError(tr("Could not load image from tile server"));
291                        }
292                        return false;
293                    }
294                } catch (UnsatisfiedLinkError | SecurityException e) {
295                    throw new IOException(e);
296                }
297            }
298        }
299        return true;
300    }
301}