001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.actions.downloadtasks;
003
004import static org.openstreetmap.josm.tools.I18n.tr;
005
006import java.io.IOException;
007import java.net.URL;
008import java.util.ArrayList;
009import java.util.Arrays;
010import java.util.Collection;
011import java.util.Collections;
012import java.util.HashSet;
013import java.util.Objects;
014import java.util.Set;
015import java.util.concurrent.Future;
016import java.util.regex.Matcher;
017import java.util.regex.Pattern;
018import java.util.stream.Stream;
019
020import org.openstreetmap.josm.data.Bounds;
021import org.openstreetmap.josm.data.DataSource;
022import org.openstreetmap.josm.data.ProjectionBounds;
023import org.openstreetmap.josm.data.ViewportData;
024import org.openstreetmap.josm.data.coor.LatLon;
025import org.openstreetmap.josm.data.osm.DataSet;
026import org.openstreetmap.josm.data.osm.OsmPrimitive;
027import org.openstreetmap.josm.data.osm.Relation;
028import org.openstreetmap.josm.data.osm.Way;
029import org.openstreetmap.josm.data.osm.visitor.BoundingXYVisitor;
030import org.openstreetmap.josm.gui.MainApplication;
031import org.openstreetmap.josm.gui.MapFrame;
032import org.openstreetmap.josm.gui.PleaseWaitRunnable;
033import org.openstreetmap.josm.gui.io.UpdatePrimitivesTask;
034import org.openstreetmap.josm.gui.layer.OsmDataLayer;
035import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
036import org.openstreetmap.josm.gui.progress.ProgressMonitor;
037import org.openstreetmap.josm.io.BoundingBoxDownloader;
038import org.openstreetmap.josm.io.OsmServerLocationReader;
039import org.openstreetmap.josm.io.OsmServerLocationReader.OsmUrlPattern;
040import org.openstreetmap.josm.io.OsmServerReader;
041import org.openstreetmap.josm.io.OsmTransferCanceledException;
042import org.openstreetmap.josm.io.OsmTransferException;
043import org.openstreetmap.josm.tools.Logging;
044import org.openstreetmap.josm.tools.Utils;
045import org.xml.sax.SAXException;
046
047/**
048 * Open the download dialog and download the data.
049 * Run in the worker thread.
050 */
051public class DownloadOsmTask extends AbstractDownloadTask<DataSet> {
052
053    protected Bounds currentBounds;
054    protected DownloadTask downloadTask;
055
056    protected String newLayerName;
057
058    /** This allows subclasses to ignore this warning */
059    protected boolean warnAboutEmptyArea = true;
060
061    @Override
062    public String[] getPatterns() {
063        if (this.getClass() == DownloadOsmTask.class) {
064            return Arrays.stream(OsmUrlPattern.values()).map(OsmUrlPattern::pattern).toArray(String[]::new);
065        } else {
066            return super.getPatterns();
067        }
068    }
069
070    @Override
071    public String getTitle() {
072        if (this.getClass() == DownloadOsmTask.class) {
073            return tr("Download OSM");
074        } else {
075            return super.getTitle();
076        }
077    }
078
079    /**
080     * Asynchronously launches the download task for a given bounding box.
081     * @param newLayer true, if the data is to be downloaded into a new layer. If false, the task
082     * selects one of the existing layers as download layer, preferably the active layer.
083     *
084     * @param downloadArea the area to download
085     * @param progressMonitor the progressMonitor
086     * @return the future representing the asynchronous task
087     * @deprecated Use {@link #download(DownloadParams, Bounds, ProgressMonitor)}
088     */
089    @Deprecated
090    public Future<?> download(boolean newLayer, Bounds downloadArea, ProgressMonitor progressMonitor) {
091        return download(new BoundingBoxDownloader(downloadArea),
092                new DownloadParams().withNewLayer(newLayer), downloadArea, progressMonitor);
093    }
094
095    @Override
096    public Future<?> download(DownloadParams settings, Bounds downloadArea, ProgressMonitor progressMonitor) {
097        return download(new BoundingBoxDownloader(downloadArea), settings, downloadArea, progressMonitor);
098    }
099
100    /**
101     * Asynchronously launches the download task for a given bounding box.
102     *
103     * @param reader the reader used to parse OSM data (see {@link OsmServerReader#parseOsm})
104     * @param newLayer newLayer true, if the data is to be downloaded into a new layer. If false, the task
105     * selects one of the existing layers as download layer, preferably the active layer.
106     * @param downloadArea the area to download
107     * @param progressMonitor the progressMonitor
108     * @return the future representing the asynchronous task
109     * @deprecated Use {@link #download(OsmServerReader, DownloadParams, Bounds, ProgressMonitor)}
110     */
111    @Deprecated
112    public Future<?> download(OsmServerReader reader, boolean newLayer, Bounds downloadArea, ProgressMonitor progressMonitor) {
113        return download(new DownloadTask(new DownloadParams().withNewLayer(newLayer), reader, progressMonitor, zoomAfterDownload),
114                downloadArea);
115    }
116
117    /**
118     * Asynchronously launches the download task for a given bounding box.
119     *
120     * Set <code>progressMonitor</code> to null, if the task should create, open, and close a progress monitor.
121     * Set progressMonitor to {@link NullProgressMonitor#INSTANCE} if progress information is to
122     * be discarded.
123     *
124     * You can wait for the asynchronous download task to finish by synchronizing on the returned
125     * {@link Future}, but make sure not to freeze up JOSM. Example:
126     * <pre>
127     *    Future&lt;?&gt; future = task.download(...);
128     *    // DON'T run this on the Swing EDT or JOSM will freeze
129     *    future.get(); // waits for the dowload task to complete
130     * </pre>
131     *
132     * The following example uses a pattern which is better suited if a task is launched from
133     * the Swing EDT:
134     * <pre>
135     *    final Future&lt;?&gt; future = task.download(...);
136     *    Runnable runAfterTask = new Runnable() {
137     *       public void run() {
138     *           // this is not strictly necessary because of the type of executor service
139     *           // Main.worker is initialized with, but it doesn't harm either
140     *           //
141     *           future.get(); // wait for the download task to complete
142     *           doSomethingAfterTheTaskCompleted();
143     *       }
144     *    }
145     *    MainApplication.worker.submit(runAfterTask);
146     * </pre>
147     * @param reader the reader used to parse OSM data (see {@link OsmServerReader#parseOsm})
148     * @param settings download settings
149     * @param downloadArea the area to download
150     * @param progressMonitor the progressMonitor
151     * @return the future representing the asynchronous task
152     * @since 13927
153     */
154    public Future<?> download(OsmServerReader reader, DownloadParams settings, Bounds downloadArea, ProgressMonitor progressMonitor) {
155        return download(new DownloadTask(settings, reader, progressMonitor, zoomAfterDownload), downloadArea);
156    }
157
158    protected Future<?> download(DownloadTask downloadTask, Bounds downloadArea) {
159        this.downloadTask = downloadTask;
160        this.currentBounds = new Bounds(downloadArea);
161        // We need submit instead of execute so we can wait for it to finish and get the error
162        // message if necessary. If no one calls getErrorMessage() it just behaves like execute.
163        return MainApplication.worker.submit(downloadTask);
164    }
165
166    /**
167     * This allows subclasses to perform operations on the URL before {@link #loadUrl} is performed.
168     * @param url the original URL
169     * @return the modified URL
170     */
171    protected String modifyUrlBeforeLoad(String url) {
172        return url;
173    }
174
175    /**
176     * Loads a given URL from the OSM Server
177     * @param settings download settings
178     * @param url The URL as String
179     */
180    @Override
181    public Future<?> loadUrl(DownloadParams settings, String url, ProgressMonitor progressMonitor) {
182        String newUrl = modifyUrlBeforeLoad(url);
183        downloadTask = new DownloadTask(settings,
184                new OsmServerLocationReader(newUrl),
185                progressMonitor);
186        currentBounds = null;
187        // Extract .osm filename from URL to set the new layer name
188        extractOsmFilename(settings, "https?://.*/(.*\\.osm)", newUrl);
189        return MainApplication.worker.submit(downloadTask);
190    }
191
192    protected final void extractOsmFilename(DownloadParams settings, String pattern, String url) {
193        newLayerName = settings.getLayerName();
194        if (newLayerName == null || newLayerName.isEmpty()) {
195            Matcher matcher = Pattern.compile(pattern).matcher(url);
196            newLayerName = matcher.matches() ? matcher.group(1) : null;
197        }
198    }
199
200    @Override
201    public void cancel() {
202        if (downloadTask != null) {
203            downloadTask.cancel();
204        }
205    }
206
207    @Override
208    public boolean isSafeForRemotecontrolRequests() {
209        return true;
210    }
211
212    @Override
213    public ProjectionBounds getDownloadProjectionBounds() {
214        return downloadTask != null ? downloadTask.computeBbox(currentBounds) : null;
215    }
216
217    /**
218     * Superclass of internal download task.
219     * @since 7636
220     */
221    public abstract static class AbstractInternalTask extends PleaseWaitRunnable {
222
223        protected final DownloadParams settings;
224        protected final boolean zoomAfterDownload;
225        protected DataSet dataSet;
226
227        /**
228         * Constructs a new {@code AbstractInternalTask}.
229         * @param settings download settings
230         * @param title message for the user
231         * @param ignoreException If true, exception will be propagated to calling code. If false then
232         * exception will be thrown directly in EDT. When this runnable is executed using executor framework
233         * then use false unless you read result of task (because exception will get lost if you don't)
234         * @param zoomAfterDownload If true, the map view will zoom to download area after download
235         */
236        public AbstractInternalTask(DownloadParams settings, String title, boolean ignoreException, boolean zoomAfterDownload) {
237            super(title, ignoreException);
238            this.settings = Objects.requireNonNull(settings);
239            this.zoomAfterDownload = zoomAfterDownload;
240        }
241
242        /**
243         * Constructs a new {@code AbstractInternalTask}.
244         * @param settings download settings
245         * @param title message for the user
246         * @param progressMonitor progress monitor
247         * @param ignoreException If true, exception will be propagated to calling code. If false then
248         * exception will be thrown directly in EDT. When this runnable is executed using executor framework
249         * then use false unless you read result of task (because exception will get lost if you don't)
250         * @param zoomAfterDownload If true, the map view will zoom to download area after download
251         */
252        public AbstractInternalTask(DownloadParams settings, String title, ProgressMonitor progressMonitor, boolean ignoreException,
253                boolean zoomAfterDownload) {
254            super(title, progressMonitor, ignoreException);
255            this.settings = Objects.requireNonNull(settings);
256            this.zoomAfterDownload = zoomAfterDownload;
257        }
258
259        protected OsmDataLayer getEditLayer() {
260            return MainApplication.getLayerManager().getEditLayer();
261        }
262
263        /**
264         * Returns the number of modifiable data layers
265         * @return number of modifiable data layers
266         * @deprecated Use {@link #getNumModifiableDataLayers}
267         */
268        @Deprecated
269        protected int getNumDataLayers() {
270            return (int) getNumModifiableDataLayers();
271        }
272
273        private static Stream<OsmDataLayer> getModifiableDataLayers() {
274            return MainApplication.getLayerManager().getLayersOfType(OsmDataLayer.class)
275                    .stream().filter(OsmDataLayer::isDownloadable);
276        }
277
278        /**
279         * Returns the number of modifiable data layers
280         * @return number of modifiable data layers
281         * @since 13434
282         */
283        protected long getNumModifiableDataLayers() {
284            return getModifiableDataLayers().count();
285        }
286
287        /**
288         * Returns the first modifiable data layer
289         * @return the first modifiable data layer
290         * @since 13434
291         */
292        protected OsmDataLayer getFirstModifiableDataLayer() {
293            return getModifiableDataLayers().findFirst().orElse(null);
294        }
295
296        protected OsmDataLayer createNewLayer(String layerName) {
297            if (layerName == null || layerName.isEmpty()) {
298                layerName = settings.getLayerName();
299            }
300            if (layerName == null || layerName.isEmpty()) {
301                layerName = OsmDataLayer.createNewName();
302            }
303            if (settings.getDownloadPolicy() != null) {
304                dataSet.setDownloadPolicy(settings.getDownloadPolicy());
305            }
306            if (settings.getUploadPolicy() != null) {
307                dataSet.setUploadPolicy(settings.getUploadPolicy());
308            }
309            if (dataSet.isLocked() && !settings.isLocked()) {
310                dataSet.unlock();
311            } else if (!dataSet.isLocked() && settings.isLocked()) {
312                dataSet.lock();
313            }
314            return new OsmDataLayer(dataSet, layerName, null);
315        }
316
317        protected OsmDataLayer createNewLayer() {
318            return createNewLayer(null);
319        }
320
321        protected ProjectionBounds computeBbox(Bounds bounds) {
322            BoundingXYVisitor v = new BoundingXYVisitor();
323            if (bounds != null) {
324                v.visit(bounds);
325            } else {
326                v.computeBoundingBox(dataSet.getNodes());
327            }
328            return v.getBounds();
329        }
330
331        protected OsmDataLayer addNewLayerIfRequired(String newLayerName) {
332            long numDataLayers = getNumModifiableDataLayers();
333            if (settings.isNewLayer() || numDataLayers == 0 || (numDataLayers > 1 && getEditLayer() == null)) {
334                // the user explicitly wants a new layer, we don't have any layer at all
335                // or it is not clear which layer to merge to
336                final OsmDataLayer layer = createNewLayer(newLayerName);
337                MainApplication.getLayerManager().addLayer(layer, zoomAfterDownload);
338                return layer;
339            }
340            return null;
341        }
342
343        protected void loadData(String newLayerName, Bounds bounds) {
344            OsmDataLayer layer = addNewLayerIfRequired(newLayerName);
345            if (layer == null) {
346                layer = getEditLayer();
347                if (layer == null || !layer.isDownloadable()) {
348                    layer = getFirstModifiableDataLayer();
349                }
350                Collection<OsmPrimitive> primitivesToUpdate = searchPrimitivesToUpdate(bounds, layer.getDataSet());
351                layer.mergeFrom(dataSet);
352                MapFrame map = MainApplication.getMap();
353                if (map != null && zoomAfterDownload && bounds != null) {
354                    map.mapView.zoomTo(new ViewportData(computeBbox(bounds)));
355                }
356                if (!primitivesToUpdate.isEmpty()) {
357                    MainApplication.worker.submit(new UpdatePrimitivesTask(layer, primitivesToUpdate));
358                }
359                layer.onPostDownloadFromServer();
360            }
361        }
362
363        /**
364         * Look for primitives deleted on server (thus absent from downloaded data)
365         * but still present in existing data layer
366         * @param bounds download bounds
367         * @param ds existing data set
368         * @return the primitives to update
369         */
370        private Collection<OsmPrimitive> searchPrimitivesToUpdate(Bounds bounds, DataSet ds) {
371            if (bounds == null)
372                return Collections.emptySet();
373            Collection<OsmPrimitive> col = new ArrayList<>();
374            ds.searchNodes(bounds.toBBox()).stream().filter(n -> !n.isNew() && !dataSet.containsNode(n)).forEachOrdered(col::add);
375            if (!col.isEmpty()) {
376                Set<Way> ways = new HashSet<>();
377                Set<Relation> rels = new HashSet<>();
378                for (OsmPrimitive n : col) {
379                    for (OsmPrimitive ref : n.getReferrers()) {
380                        if (ref.isNew()) {
381                            continue;
382                        } else if (ref instanceof Way) {
383                            ways.add((Way) ref);
384                        } else if (ref instanceof Relation) {
385                            rels.add((Relation) ref);
386                        }
387                    }
388                }
389                ways.stream().filter(w -> !dataSet.containsWay(w)).forEachOrdered(col::add);
390                rels.stream().filter(r -> !dataSet.containsRelation(r)).forEachOrdered(col::add);
391            }
392            return col;
393        }
394    }
395
396    protected class DownloadTask extends AbstractInternalTask {
397        protected final OsmServerReader reader;
398
399        /**
400         * Constructs a new {@code DownloadTask}.
401         * @param newLayer if {@code true}, force download to a new layer
402         * @param reader OSM data reader
403         * @param progressMonitor progress monitor
404         * @deprecated Use {@code DownloadTask(DownloadParams, OsmServerReader, ProgressMonitor)}
405         */
406        @Deprecated
407        public DownloadTask(boolean newLayer, OsmServerReader reader, ProgressMonitor progressMonitor) {
408            this(new DownloadParams().withNewLayer(newLayer), reader, progressMonitor);
409        }
410
411        /**
412         * Constructs a new {@code DownloadTask}.
413         * @param newLayer if {@code true}, force download to a new layer
414         * @param reader OSM data reader
415         * @param progressMonitor progress monitor
416         * @param zoomAfterDownload If true, the map view will zoom to download area after download
417         * @deprecated Use {@code DownloadTask(DownloadParams, OsmServerReader, ProgressMonitor, boolean)}
418         */
419        @Deprecated
420        public DownloadTask(boolean newLayer, OsmServerReader reader, ProgressMonitor progressMonitor, boolean zoomAfterDownload) {
421            this(new DownloadParams().withNewLayer(newLayer), reader, progressMonitor, zoomAfterDownload);
422        }
423
424        /**
425         * Constructs a new {@code DownloadTask}.
426         * @param settings download settings
427         * @param reader OSM data reader
428         * @param progressMonitor progress monitor
429         * @since 13927
430         */
431        public DownloadTask(DownloadParams settings, OsmServerReader reader, ProgressMonitor progressMonitor) {
432            this(settings, reader, progressMonitor, true);
433        }
434
435        /**
436         * Constructs a new {@code DownloadTask}.
437         * @param settings download settings
438         * @param reader OSM data reader
439         * @param progressMonitor progress monitor
440         * @param zoomAfterDownload If true, the map view will zoom to download area after download
441         * @since 13927
442         */
443        public DownloadTask(DownloadParams settings, OsmServerReader reader, ProgressMonitor progressMonitor, boolean zoomAfterDownload) {
444            super(settings, tr("Downloading data"), progressMonitor, false, zoomAfterDownload);
445            this.reader = reader;
446        }
447
448        protected DataSet parseDataSet() throws OsmTransferException {
449            return reader.parseOsm(progressMonitor.createSubTaskMonitor(ProgressMonitor.ALL_TICKS, false));
450        }
451
452        @Override
453        public void realRun() throws IOException, SAXException, OsmTransferException {
454            try {
455                if (isCanceled())
456                    return;
457                dataSet = parseDataSet();
458            } catch (OsmTransferException e) {
459                if (isCanceled()) {
460                    Logging.info(tr("Ignoring exception because download has been canceled. Exception was: {0}", e.toString()));
461                    return;
462                }
463                if (e instanceof OsmTransferCanceledException) {
464                    setCanceled(true);
465                    return;
466                } else {
467                    rememberException(e);
468                }
469                DownloadOsmTask.this.setFailed(true);
470            }
471        }
472
473        @Override
474        protected void finish() {
475            if (isFailed() || isCanceled())
476                return;
477            if (dataSet == null)
478                return; // user canceled download or error occurred
479            if (dataSet.allPrimitives().isEmpty()) {
480                if (warnAboutEmptyArea) {
481                    rememberErrorMessage(tr("No data found in this area."));
482                }
483                // need to synthesize a download bounds lest the visual indication of downloaded area doesn't work
484                dataSet.addDataSource(new DataSource(currentBounds != null ? currentBounds :
485                    new Bounds(LatLon.ZERO), "OpenStreetMap server"));
486            }
487
488            rememberDownloadedData(dataSet);
489            loadData(newLayerName, currentBounds);
490        }
491
492        @Override
493        protected void cancel() {
494            setCanceled(true);
495            if (reader != null) {
496                reader.cancel();
497            }
498        }
499    }
500
501    @Override
502    public String getConfirmationMessage(URL url) {
503        if (url != null) {
504            String urlString = url.toExternalForm();
505            if (urlString.matches(OsmUrlPattern.OSM_API_URL.pattern())) {
506                // TODO: proper i18n after stabilization
507                Collection<String> items = new ArrayList<>();
508                items.add(tr("OSM Server URL:") + ' ' + url.getHost());
509                items.add(tr("Command")+": "+url.getPath());
510                if (url.getQuery() != null) {
511                    items.add(tr("Request details: {0}", url.getQuery().replaceAll(",\\s*", ", ")));
512                }
513                return Utils.joinAsHtmlUnorderedList(items);
514            }
515            // TODO: other APIs
516        }
517        return null;
518    }
519}