001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.data.imagery;
003
004import static org.openstreetmap.josm.tools.I18n.tr;
005
006import java.io.ByteArrayInputStream;
007import java.io.IOException;
008import java.net.URL;
009import java.nio.charset.StandardCharsets;
010import java.util.HashSet;
011import java.util.List;
012import java.util.Map;
013import java.util.Map.Entry;
014import java.util.Optional;
015import java.util.Set;
016import java.util.concurrent.ConcurrentHashMap;
017import java.util.concurrent.ConcurrentMap;
018import java.util.concurrent.ThreadPoolExecutor;
019import java.util.concurrent.TimeUnit;
020import java.util.regex.Matcher;
021import java.util.regex.Pattern;
022
023import org.apache.commons.jcs.access.behavior.ICacheAccess;
024import org.openstreetmap.gui.jmapviewer.Tile;
025import org.openstreetmap.gui.jmapviewer.interfaces.TileJob;
026import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener;
027import org.openstreetmap.gui.jmapviewer.interfaces.TileSource;
028import org.openstreetmap.gui.jmapviewer.tilesources.AbstractTMSTileSource;
029import org.openstreetmap.josm.data.cache.BufferedImageCacheEntry;
030import org.openstreetmap.josm.data.cache.CacheEntry;
031import org.openstreetmap.josm.data.cache.CacheEntryAttributes;
032import org.openstreetmap.josm.data.cache.ICachedLoaderListener;
033import org.openstreetmap.josm.data.cache.JCSCachedTileLoaderJob;
034import org.openstreetmap.josm.data.preferences.LongProperty;
035import org.openstreetmap.josm.tools.HttpClient;
036import org.openstreetmap.josm.tools.Logging;
037import org.openstreetmap.josm.tools.Utils;
038
039/**
040 * Class bridging TMS requests to JCS cache requests
041 *
042 * @author Wiktor Niesiobędzki
043 * @since 8168
044 */
045public class TMSCachedTileLoaderJob extends JCSCachedTileLoaderJob<String, BufferedImageCacheEntry> implements TileJob, ICachedLoaderListener {
046    private static final LongProperty MAXIMUM_EXPIRES = new LongProperty("imagery.generic.maximum_expires", TimeUnit.DAYS.toMillis(30));
047    private static final LongProperty MINIMUM_EXPIRES = new LongProperty("imagery.generic.minimum_expires", TimeUnit.HOURS.toMillis(1));
048    static final Pattern SERVICE_EXCEPTION_PATTERN = Pattern.compile("(?s).+<ServiceException[^>]*>(.+)</ServiceException>.+");
049    protected final Tile tile;
050    private volatile URL url;
051
052    // we need another deduplication of Tile Loader listeners, as for each submit, new TMSCachedTileLoaderJob was created
053    // that way, we reduce calls to tileLoadingFinished, and general CPU load due to surplus Map repaints
054    private static final ConcurrentMap<String, Set<TileLoaderListener>> inProgress = new ConcurrentHashMap<>();
055
056    /**
057     * Constructor for creating a job, to get a specific tile from cache
058     * @param listener Tile loader listener
059     * @param tile to be fetched from cache
060     * @param cache object
061     * @param connectTimeout when connecting to remote resource
062     * @param readTimeout when connecting to remote resource
063     * @param headers HTTP headers to be sent together with request
064     * @param downloadExecutor that will be executing the jobs
065     */
066    public TMSCachedTileLoaderJob(TileLoaderListener listener, Tile tile,
067            ICacheAccess<String, BufferedImageCacheEntry> cache,
068            int connectTimeout, int readTimeout, Map<String, String> headers,
069            ThreadPoolExecutor downloadExecutor) {
070        super(cache, connectTimeout, readTimeout, headers, downloadExecutor);
071        this.tile = tile;
072        if (listener != null) {
073            String deduplicationKey = getCacheKey();
074            synchronized (inProgress) {
075                inProgress.computeIfAbsent(deduplicationKey, k -> new HashSet<>()).add(listener);
076            }
077        }
078    }
079
080    @Override
081    public String getCacheKey() {
082        if (tile != null) {
083            TileSource tileSource = tile.getTileSource();
084            return Optional.ofNullable(tileSource.getName()).orElse("").replace(':', '_') + ':'
085                    + tileSource.getTileId(tile.getZoom(), tile.getXtile(), tile.getYtile());
086        }
087        return null;
088    }
089
090    /*
091     *  this doesn't needs to be synchronized, as it's not that costly to keep only one execution
092     *  in parallel, but URL creation and Tile.getUrl() are costly and are not needed when fetching
093     *  data from cache, that's why URL creation is postponed until it's needed
094     *
095     *  We need to have static url value for TileLoaderJob, as for some TileSources we might get different
096     *  URL's each call we made (servers switching), and URL's are used below as a key for duplicate detection
097     *
098     */
099    @Override
100    public URL getUrl() throws IOException {
101        if (url == null) {
102            synchronized (this) {
103                if (url == null) {
104                    String sUrl = tile.getUrl();
105                    if (!"".equals(sUrl)) {
106                        url = new URL(sUrl);
107                    }
108                }
109            }
110        }
111        return url;
112    }
113
114    @Override
115    public boolean isObjectLoadable() {
116        if (cacheData != null) {
117            byte[] content = cacheData.getContent();
118            try {
119                return content.length > 0 || cacheData.getImage() != null || isNoTileAtZoom();
120            } catch (IOException e) {
121                Logging.logWithStackTrace(Logging.LEVEL_WARN, e, "JCS TMS - error loading from cache for tile {0}: {1}",
122                        new Object[] {tile.getKey(), e.getMessage()}
123                        );
124            }
125        }
126        return false;
127    }
128
129    @Override
130    protected boolean isResponseLoadable(Map<String, List<String>> headers, int statusCode, byte[] content) {
131        attributes.setMetadata(tile.getTileSource().getMetadata(headers));
132        if (tile.getTileSource().isNoTileAtZoom(headers, statusCode, content)) {
133            attributes.setNoTileAtZoom(true);
134            return false; // do no try to load data from no-tile at zoom, cache empty object instead
135        }
136        return super.isResponseLoadable(headers, statusCode, content);
137    }
138
139    @Override
140    protected boolean cacheAsEmpty() {
141        return isNoTileAtZoom() || super.cacheAsEmpty();
142    }
143
144    @Override
145    public void submit(boolean force) {
146        tile.initLoading();
147        try {
148            super.submit(this, force);
149        } catch (IOException | IllegalArgumentException e) {
150            // if we fail to submit the job, mark tile as loaded and set error message
151            Logging.log(Logging.LEVEL_WARN, e);
152            tile.finishLoading();
153            tile.setError(e.getMessage());
154        }
155    }
156
157    @Override
158    public void loadingFinished(CacheEntry object, CacheEntryAttributes attributes, LoadResult result) {
159        this.attributes = attributes; // as we might get notification from other object than our selfs, pass attributes along
160        Set<TileLoaderListener> listeners;
161        synchronized (inProgress) {
162            listeners = inProgress.remove(getCacheKey());
163        }
164        boolean status = result.equals(LoadResult.SUCCESS);
165
166        try {
167            tile.finishLoading(); // whatever happened set that loading has finished
168            // set tile metadata
169            if (this.attributes != null) {
170                for (Entry<String, String> e: this.attributes.getMetadata().entrySet()) {
171                    tile.putValue(e.getKey(), e.getValue());
172                }
173            }
174
175            switch(result) {
176            case SUCCESS:
177                handleNoTileAtZoom();
178                if (attributes != null) {
179                    int httpStatusCode = attributes.getResponseCode();
180                    if (httpStatusCode >= 400 && !isNoTileAtZoom()) {
181                        if (attributes.getErrorMessage() == null) {
182                            tile.setError(tr("HTTP error {0} when loading tiles", httpStatusCode));
183                        } else {
184                            tile.setError(tr("Error downloading tiles: {0}", attributes.getErrorMessage()));
185                        }
186                        status = false;
187                    }
188                }
189                status &= tryLoadTileImage(object); //try to keep returned image as background
190                break;
191            case FAILURE:
192                tile.setError("Problem loading tile");
193                tryLoadTileImage(object);
194                break;
195            case CANCELED:
196                tile.loadingCanceled();
197                // do nothing
198            }
199
200            // always check, if there is some listener interested in fact, that tile has finished loading
201            if (listeners != null) { // listeners might be null, if some other thread notified already about success
202                for (TileLoaderListener l: listeners) {
203                    l.tileLoadingFinished(tile, status);
204                }
205            }
206        } catch (IOException e) {
207            Logging.warn("JCS TMS - error loading object for tile {0}: {1}", new Object[] {tile.getKey(), e.getMessage()});
208            tile.setError(e);
209            tile.setLoaded(false);
210            if (listeners != null) { // listeners might be null, if some other thread notified already about success
211                for (TileLoaderListener l: listeners) {
212                    l.tileLoadingFinished(tile, false);
213                }
214            }
215        }
216    }
217
218    /**
219     * For TMS use BaseURL as settings discovery, so for different paths, we will have different settings (useful for developer servers)
220     *
221     * @return base URL of TMS or server url as defined in super class
222     */
223    @Override
224    protected String getServerKey() {
225        TileSource ts = tile.getSource();
226        if (ts instanceof AbstractTMSTileSource) {
227            return ((AbstractTMSTileSource) ts).getBaseUrl();
228        }
229        return super.getServerKey();
230    }
231
232    @Override
233    protected BufferedImageCacheEntry createCacheEntry(byte[] content) {
234        return new BufferedImageCacheEntry(content);
235    }
236
237    @Override
238    public void submit() {
239        submit(false);
240    }
241
242    @Override
243    protected CacheEntryAttributes parseHeaders(HttpClient.Response urlConn) {
244        CacheEntryAttributes ret = super.parseHeaders(urlConn);
245        // keep the expiration time between MINIMUM_EXPIRES and MAXIMUM_EXPIRES, so we will cache the tiles
246        // at least for some short period of time, but not too long
247        if (ret.getExpirationTime() < now + MINIMUM_EXPIRES.get()) {
248            ret.setExpirationTime(now + MINIMUM_EXPIRES.get());
249        }
250        if (ret.getExpirationTime() > now + MAXIMUM_EXPIRES.get()) {
251            ret.setExpirationTime(now + MAXIMUM_EXPIRES.get());
252        }
253        return ret;
254    }
255
256    private boolean handleNoTileAtZoom() {
257        if (isNoTileAtZoom()) {
258            Logging.debug("JCS TMS - Tile valid, but no file, as no tiles at this level {0}", tile);
259            tile.setError("No tile at this zoom level");
260            tile.putValue("tile-info", "no-tile");
261            return true;
262        }
263        return false;
264    }
265
266    private boolean isNoTileAtZoom() {
267        if (attributes == null) {
268            Logging.warn("Cache attributes are null");
269        }
270        return attributes != null && attributes.isNoTileAtZoom();
271    }
272
273    private boolean tryLoadTileImage(CacheEntry object) throws IOException {
274        if (object != null) {
275            byte[] content = object.getContent();
276            if (content.length > 0) {
277                try (ByteArrayInputStream in = new ByteArrayInputStream(content)) {
278                    tile.loadImage(in);
279                    if (tile.getImage() == null) {
280                        String s = new String(content, StandardCharsets.UTF_8);
281                        Matcher m = SERVICE_EXCEPTION_PATTERN.matcher(s);
282                        if (m.matches()) {
283                            String message = Utils.strip(m.group(1));
284                            tile.setError(message);
285                            Logging.error(message);
286                            Logging.debug(s);
287                        } else {
288                            tile.setError(tr("Could not load image from tile server"));
289                        }
290                        return false;
291                    }
292                } catch (UnsatisfiedLinkError e) {
293                    throw new IOException(e);
294                }
295            }
296        }
297        return true;
298    }
299}