001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.actions.downloadtasks;
003
004import static org.openstreetmap.josm.tools.I18n.tr;
005
006import java.io.IOException;
007import java.net.URL;
008import java.util.ArrayList;
009import java.util.Arrays;
010import java.util.Collection;
011import java.util.Collections;
012import java.util.HashSet;
013import java.util.Set;
014import java.util.concurrent.Future;
015import java.util.regex.Matcher;
016import java.util.regex.Pattern;
017import java.util.stream.Stream;
018
019import org.openstreetmap.josm.data.Bounds;
020import org.openstreetmap.josm.data.DataSource;
021import org.openstreetmap.josm.data.ProjectionBounds;
022import org.openstreetmap.josm.data.ViewportData;
023import org.openstreetmap.josm.data.coor.LatLon;
024import org.openstreetmap.josm.data.osm.DataSet;
025import org.openstreetmap.josm.data.osm.OsmPrimitive;
026import org.openstreetmap.josm.data.osm.Relation;
027import org.openstreetmap.josm.data.osm.Way;
028import org.openstreetmap.josm.data.osm.visitor.BoundingXYVisitor;
029import org.openstreetmap.josm.gui.MainApplication;
030import org.openstreetmap.josm.gui.MapFrame;
031import org.openstreetmap.josm.gui.PleaseWaitRunnable;
032import org.openstreetmap.josm.gui.io.UpdatePrimitivesTask;
033import org.openstreetmap.josm.gui.layer.OsmDataLayer;
034import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
035import org.openstreetmap.josm.gui.progress.ProgressMonitor;
036import org.openstreetmap.josm.io.BoundingBoxDownloader;
037import org.openstreetmap.josm.io.OsmServerLocationReader;
038import org.openstreetmap.josm.io.OsmServerLocationReader.OsmUrlPattern;
039import org.openstreetmap.josm.io.OsmServerReader;
040import org.openstreetmap.josm.io.OsmTransferCanceledException;
041import org.openstreetmap.josm.io.OsmTransferException;
042import org.openstreetmap.josm.tools.Logging;
043import org.openstreetmap.josm.tools.Utils;
044import org.xml.sax.SAXException;
045
046/**
047 * Open the download dialog and download the data.
048 * Run in the worker thread.
049 */
050public class DownloadOsmTask extends AbstractDownloadTask<DataSet> {
051
052    protected Bounds currentBounds;
053    protected DownloadTask downloadTask;
054
055    protected String newLayerName;
056
057    /** This allows subclasses to ignore this warning */
058    protected boolean warnAboutEmptyArea = true;
059
060    @Override
061    public String[] getPatterns() {
062        if (this.getClass() == DownloadOsmTask.class) {
063            return Arrays.stream(OsmUrlPattern.values()).map(OsmUrlPattern::pattern).toArray(String[]::new);
064        } else {
065            return super.getPatterns();
066        }
067    }
068
069    @Override
070    public String getTitle() {
071        if (this.getClass() == DownloadOsmTask.class) {
072            return tr("Download OSM");
073        } else {
074            return super.getTitle();
075        }
076    }
077
078    @Override
079    public Future<?> download(boolean newLayer, Bounds downloadArea, ProgressMonitor progressMonitor) {
080        return download(new BoundingBoxDownloader(downloadArea), newLayer, downloadArea, progressMonitor);
081    }
082
083    /**
084     * Asynchronously launches the download task for a given bounding box.
085     *
086     * Set <code>progressMonitor</code> to null, if the task should create, open, and close a progress monitor.
087     * Set progressMonitor to {@link NullProgressMonitor#INSTANCE} if progress information is to
088     * be discarded.
089     *
090     * You can wait for the asynchronous download task to finish by synchronizing on the returned
091     * {@link Future}, but make sure not to freeze up JOSM. Example:
092     * <pre>
093     *    Future&lt;?&gt; future = task.download(...);
094     *    // DON'T run this on the Swing EDT or JOSM will freeze
095     *    future.get(); // waits for the dowload task to complete
096     * </pre>
097     *
098     * The following example uses a pattern which is better suited if a task is launched from
099     * the Swing EDT:
100     * <pre>
101     *    final Future&lt;?&gt; future = task.download(...);
102     *    Runnable runAfterTask = new Runnable() {
103     *       public void run() {
104     *           // this is not strictly necessary because of the type of executor service
105     *           // Main.worker is initialized with, but it doesn't harm either
106     *           //
107     *           future.get(); // wait for the download task to complete
108     *           doSomethingAfterTheTaskCompleted();
109     *       }
110     *    }
111     *    MainApplication.worker.submit(runAfterTask);
112     * </pre>
113     * @param reader the reader used to parse OSM data (see {@link OsmServerReader#parseOsm})
114     * @param newLayer true, if the data is to be downloaded into a new layer. If false, the task
115     *                 selects one of the existing layers as download layer, preferably the active layer.
116     * @param downloadArea the area to download
117     * @param progressMonitor the progressMonitor
118     * @return the future representing the asynchronous task
119     */
120    public Future<?> download(OsmServerReader reader, boolean newLayer, Bounds downloadArea, ProgressMonitor progressMonitor) {
121        return download(new DownloadTask(newLayer, reader, progressMonitor, zoomAfterDownload), downloadArea);
122    }
123
124    protected Future<?> download(DownloadTask downloadTask, Bounds downloadArea) {
125        this.downloadTask = downloadTask;
126        this.currentBounds = new Bounds(downloadArea);
127        // We need submit instead of execute so we can wait for it to finish and get the error
128        // message if necessary. If no one calls getErrorMessage() it just behaves like execute.
129        return MainApplication.worker.submit(downloadTask);
130    }
131
132    /**
133     * This allows subclasses to perform operations on the URL before {@link #loadUrl} is performed.
134     * @param url the original URL
135     * @return the modified URL
136     */
137    protected String modifyUrlBeforeLoad(String url) {
138        return url;
139    }
140
141    /**
142     * Loads a given URL from the OSM Server
143     * @param newLayer True if the data should be saved to a new layer
144     * @param url The URL as String
145     */
146    @Override
147    public Future<?> loadUrl(boolean newLayer, String url, ProgressMonitor progressMonitor) {
148        String newUrl = modifyUrlBeforeLoad(url);
149        downloadTask = new DownloadTask(newLayer,
150                new OsmServerLocationReader(newUrl),
151                progressMonitor);
152        currentBounds = null;
153        // Extract .osm filename from URL to set the new layer name
154        extractOsmFilename("https?://.*/(.*\\.osm)", newUrl);
155        return MainApplication.worker.submit(downloadTask);
156    }
157
158    protected final void extractOsmFilename(String pattern, String url) {
159        Matcher matcher = Pattern.compile(pattern).matcher(url);
160        newLayerName = matcher.matches() ? matcher.group(1) : null;
161    }
162
163    @Override
164    public void cancel() {
165        if (downloadTask != null) {
166            downloadTask.cancel();
167        }
168    }
169
170    @Override
171    public boolean isSafeForRemotecontrolRequests() {
172        return true;
173    }
174
175    @Override
176    public ProjectionBounds getDownloadProjectionBounds() {
177        return downloadTask != null ? downloadTask.computeBbox(currentBounds) : null;
178    }
179
180    /**
181     * Superclass of internal download task.
182     * @since 7636
183     */
184    public abstract static class AbstractInternalTask extends PleaseWaitRunnable {
185
186        protected final boolean newLayer;
187        protected final boolean zoomAfterDownload;
188        protected DataSet dataSet;
189
190        /**
191         * Constructs a new {@code AbstractInternalTask}.
192         * @param newLayer if {@code true}, force download to a new layer
193         * @param title message for the user
194         * @param ignoreException If true, exception will be propagated to calling code. If false then
195         * exception will be thrown directly in EDT. When this runnable is executed using executor framework
196         * then use false unless you read result of task (because exception will get lost if you don't)
197         * @param zoomAfterDownload If true, the map view will zoom to download area after download
198         */
199        public AbstractInternalTask(boolean newLayer, String title, boolean ignoreException, boolean zoomAfterDownload) {
200            super(title, ignoreException);
201            this.newLayer = newLayer;
202            this.zoomAfterDownload = zoomAfterDownload;
203        }
204
205        /**
206         * Constructs a new {@code AbstractInternalTask}.
207         * @param newLayer if {@code true}, force download to a new layer
208         * @param title message for the user
209         * @param progressMonitor progress monitor
210         * @param ignoreException If true, exception will be propagated to calling code. If false then
211         * exception will be thrown directly in EDT. When this runnable is executed using executor framework
212         * then use false unless you read result of task (because exception will get lost if you don't)
213         * @param zoomAfterDownload If true, the map view will zoom to download area after download
214         */
215        public AbstractInternalTask(boolean newLayer, String title, ProgressMonitor progressMonitor, boolean ignoreException,
216                boolean zoomAfterDownload) {
217            super(title, progressMonitor, ignoreException);
218            this.newLayer = newLayer;
219            this.zoomAfterDownload = zoomAfterDownload;
220        }
221
222        protected OsmDataLayer getEditLayer() {
223            return MainApplication.getLayerManager().getEditLayer();
224        }
225
226        /**
227         * Returns the number of modifiable data layers
228         * @return number of modifiable data layers
229         * @deprecated Use {@link #getNumModifiableDataLayers}
230         */
231        @Deprecated
232        protected int getNumDataLayers() {
233            return (int) getNumModifiableDataLayers();
234        }
235
236        private static Stream<OsmDataLayer> getModifiableDataLayers() {
237            return MainApplication.getLayerManager().getLayersOfType(OsmDataLayer.class)
238                    .stream().filter(OsmDataLayer::isDownloadable);
239        }
240
241        /**
242         * Returns the number of modifiable data layers
243         * @return number of modifiable data layers
244         * @since 13434
245         */
246        protected long getNumModifiableDataLayers() {
247            return getModifiableDataLayers().count();
248        }
249
250        /**
251         * Returns the first modifiable data layer
252         * @return the first modifiable data layer
253         * @since 13434
254         */
255        protected OsmDataLayer getFirstModifiableDataLayer() {
256            return getModifiableDataLayers().findFirst().orElse(null);
257        }
258
259        protected OsmDataLayer createNewLayer(String layerName) {
260            if (layerName == null || layerName.isEmpty()) {
261                layerName = OsmDataLayer.createNewName();
262            }
263            return new OsmDataLayer(dataSet, layerName, null);
264        }
265
266        protected OsmDataLayer createNewLayer() {
267            return createNewLayer(null);
268        }
269
270        protected ProjectionBounds computeBbox(Bounds bounds) {
271            BoundingXYVisitor v = new BoundingXYVisitor();
272            if (bounds != null) {
273                v.visit(bounds);
274            } else {
275                v.computeBoundingBox(dataSet.getNodes());
276            }
277            return v.getBounds();
278        }
279
280        protected OsmDataLayer addNewLayerIfRequired(String newLayerName) {
281            long numDataLayers = getNumModifiableDataLayers();
282            if (newLayer || numDataLayers == 0 || (numDataLayers > 1 && getEditLayer() == null)) {
283                // the user explicitly wants a new layer, we don't have any layer at all
284                // or it is not clear which layer to merge to
285                final OsmDataLayer layer = createNewLayer(newLayerName);
286                MainApplication.getLayerManager().addLayer(layer, zoomAfterDownload);
287                return layer;
288            }
289            return null;
290        }
291
292        protected void loadData(String newLayerName, Bounds bounds) {
293            OsmDataLayer layer = addNewLayerIfRequired(newLayerName);
294            if (layer == null) {
295                layer = getEditLayer();
296                if (layer == null || !layer.isDownloadable()) {
297                    layer = getFirstModifiableDataLayer();
298                }
299                Collection<OsmPrimitive> primitivesToUpdate = searchPrimitivesToUpdate(bounds, layer.getDataSet());
300                layer.mergeFrom(dataSet);
301                MapFrame map = MainApplication.getMap();
302                if (map != null && zoomAfterDownload && bounds != null) {
303                    map.mapView.zoomTo(new ViewportData(computeBbox(bounds)));
304                }
305                if (!primitivesToUpdate.isEmpty()) {
306                    MainApplication.worker.submit(new UpdatePrimitivesTask(layer, primitivesToUpdate));
307                }
308                layer.onPostDownloadFromServer();
309            }
310        }
311
312        /**
313         * Look for primitives deleted on server (thus absent from downloaded data)
314         * but still present in existing data layer
315         * @param bounds download bounds
316         * @param ds existing data set
317         * @return the primitives to update
318         */
319        private Collection<OsmPrimitive> searchPrimitivesToUpdate(Bounds bounds, DataSet ds) {
320            if (bounds == null)
321                return Collections.emptySet();
322            Collection<OsmPrimitive> col = new ArrayList<>();
323            ds.searchNodes(bounds.toBBox()).stream().filter(n -> !n.isNew() && !dataSet.containsNode(n)).forEachOrdered(col::add);
324            if (!col.isEmpty()) {
325                Set<Way> ways = new HashSet<>();
326                Set<Relation> rels = new HashSet<>();
327                for (OsmPrimitive n : col) {
328                    for (OsmPrimitive ref : n.getReferrers()) {
329                        if (ref.isNew()) {
330                            continue;
331                        } else if (ref instanceof Way) {
332                            ways.add((Way) ref);
333                        } else if (ref instanceof Relation) {
334                            rels.add((Relation) ref);
335                        }
336                    }
337                }
338                ways.stream().filter(w -> !dataSet.containsWay(w)).forEachOrdered(col::add);
339                rels.stream().filter(r -> !dataSet.containsRelation(r)).forEachOrdered(col::add);
340            }
341            return col;
342        }
343    }
344
345    protected class DownloadTask extends AbstractInternalTask {
346        protected final OsmServerReader reader;
347
348        /**
349         * Constructs a new {@code DownloadTask}.
350         * @param newLayer if {@code true}, force download to a new layer
351         * @param reader OSM data reader
352         * @param progressMonitor progress monitor
353         */
354        public DownloadTask(boolean newLayer, OsmServerReader reader, ProgressMonitor progressMonitor) {
355            this(newLayer, reader, progressMonitor, true);
356        }
357
358        /**
359         * Constructs a new {@code DownloadTask}.
360         * @param newLayer if {@code true}, force download to a new layer
361         * @param reader OSM data reader
362         * @param progressMonitor progress monitor
363         * @param zoomAfterDownload If true, the map view will zoom to download area after download
364         * @since 8942
365         */
366        public DownloadTask(boolean newLayer, OsmServerReader reader, ProgressMonitor progressMonitor, boolean zoomAfterDownload) {
367            super(newLayer, tr("Downloading data"), progressMonitor, false, zoomAfterDownload);
368            this.reader = reader;
369        }
370
371        protected DataSet parseDataSet() throws OsmTransferException {
372            return reader.parseOsm(progressMonitor.createSubTaskMonitor(ProgressMonitor.ALL_TICKS, false));
373        }
374
375        @Override
376        public void realRun() throws IOException, SAXException, OsmTransferException {
377            try {
378                if (isCanceled())
379                    return;
380                dataSet = parseDataSet();
381            } catch (OsmTransferException e) {
382                if (isCanceled()) {
383                    Logging.info(tr("Ignoring exception because download has been canceled. Exception was: {0}", e.toString()));
384                    return;
385                }
386                if (e instanceof OsmTransferCanceledException) {
387                    setCanceled(true);
388                    return;
389                } else {
390                    rememberException(e);
391                }
392                DownloadOsmTask.this.setFailed(true);
393            }
394        }
395
396        @Override
397        protected void finish() {
398            if (isFailed() || isCanceled())
399                return;
400            if (dataSet == null)
401                return; // user canceled download or error occurred
402            if (dataSet.allPrimitives().isEmpty()) {
403                if (warnAboutEmptyArea) {
404                    rememberErrorMessage(tr("No data found in this area."));
405                }
406                // need to synthesize a download bounds lest the visual indication of downloaded area doesn't work
407                dataSet.addDataSource(new DataSource(currentBounds != null ? currentBounds :
408                    new Bounds(LatLon.ZERO), "OpenStreetMap server"));
409            }
410
411            rememberDownloadedData(dataSet);
412            loadData(newLayerName, currentBounds);
413        }
414
415        @Override
416        protected void cancel() {
417            setCanceled(true);
418            if (reader != null) {
419                reader.cancel();
420            }
421        }
422    }
423
424    @Override
425    public String getConfirmationMessage(URL url) {
426        if (url != null) {
427            String urlString = url.toExternalForm();
428            if (urlString.matches(OsmUrlPattern.OSM_API_URL.pattern())) {
429                // TODO: proper i18n after stabilization
430                Collection<String> items = new ArrayList<>();
431                items.add(tr("OSM Server URL:") + ' ' + url.getHost());
432                items.add(tr("Command")+": "+url.getPath());
433                if (url.getQuery() != null) {
434                    items.add(tr("Request details: {0}", url.getQuery().replaceAll(",\\s*", ", ")));
435                }
436                return Utils.joinAsHtmlUnorderedList(items);
437            }
438            // TODO: other APIs
439        }
440        return null;
441    }
442}