001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.actions.downloadtasks; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.io.IOException; 007import java.net.URL; 008import java.util.ArrayList; 009import java.util.Arrays; 010import java.util.Collection; 011import java.util.Collections; 012import java.util.HashSet; 013import java.util.Objects; 014import java.util.Set; 015import java.util.concurrent.Future; 016import java.util.regex.Matcher; 017import java.util.regex.Pattern; 018import java.util.stream.Stream; 019 020import org.openstreetmap.josm.data.Bounds; 021import org.openstreetmap.josm.data.DataSource; 022import org.openstreetmap.josm.data.ProjectionBounds; 023import org.openstreetmap.josm.data.ViewportData; 024import org.openstreetmap.josm.data.coor.LatLon; 025import org.openstreetmap.josm.data.osm.DataSet; 026import org.openstreetmap.josm.data.osm.OsmPrimitive; 027import org.openstreetmap.josm.data.osm.Relation; 028import org.openstreetmap.josm.data.osm.Way; 029import org.openstreetmap.josm.data.osm.visitor.BoundingXYVisitor; 030import org.openstreetmap.josm.gui.MainApplication; 031import org.openstreetmap.josm.gui.MapFrame; 032import org.openstreetmap.josm.gui.PleaseWaitRunnable; 033import org.openstreetmap.josm.gui.io.UpdatePrimitivesTask; 034import org.openstreetmap.josm.gui.layer.OsmDataLayer; 035import org.openstreetmap.josm.gui.progress.NullProgressMonitor; 036import org.openstreetmap.josm.gui.progress.ProgressMonitor; 037import org.openstreetmap.josm.io.BoundingBoxDownloader; 038import org.openstreetmap.josm.io.OsmServerLocationReader; 039import org.openstreetmap.josm.io.OsmServerLocationReader.OsmUrlPattern; 040import org.openstreetmap.josm.io.OsmServerReader; 041import org.openstreetmap.josm.io.OsmTransferCanceledException; 042import org.openstreetmap.josm.io.OsmTransferException; 043import org.openstreetmap.josm.tools.Logging; 044import org.openstreetmap.josm.tools.Utils; 045import org.xml.sax.SAXException; 046 047/** 048 * Open the download dialog and download the data. 049 * Run in the worker thread. 050 */ 051public class DownloadOsmTask extends AbstractDownloadTask<DataSet> { 052 053 protected Bounds currentBounds; 054 protected DownloadTask downloadTask; 055 056 protected String newLayerName; 057 058 /** This allows subclasses to ignore this warning */ 059 protected boolean warnAboutEmptyArea = true; 060 061 @Override 062 public String[] getPatterns() { 063 if (this.getClass() == DownloadOsmTask.class) { 064 return Arrays.stream(OsmUrlPattern.values()).map(OsmUrlPattern::pattern).toArray(String[]::new); 065 } else { 066 return super.getPatterns(); 067 } 068 } 069 070 @Override 071 public String getTitle() { 072 if (this.getClass() == DownloadOsmTask.class) { 073 return tr("Download OSM"); 074 } else { 075 return super.getTitle(); 076 } 077 } 078 079 /** 080 * Asynchronously launches the download task for a given bounding box. 081 * @param newLayer true, if the data is to be downloaded into a new layer. If false, the task 082 * selects one of the existing layers as download layer, preferably the active layer. 083 * 084 * @param downloadArea the area to download 085 * @param progressMonitor the progressMonitor 086 * @return the future representing the asynchronous task 087 * @deprecated Use {@link #download(DownloadParams, Bounds, ProgressMonitor)} 088 */ 089 @Deprecated 090 public Future<?> download(boolean newLayer, Bounds downloadArea, ProgressMonitor progressMonitor) { 091 return download(new BoundingBoxDownloader(downloadArea), 092 new DownloadParams().withNewLayer(newLayer), downloadArea, progressMonitor); 093 } 094 095 @Override 096 public Future<?> download(DownloadParams settings, Bounds downloadArea, ProgressMonitor progressMonitor) { 097 return download(new BoundingBoxDownloader(downloadArea), settings, downloadArea, progressMonitor); 098 } 099 100 /** 101 * Asynchronously launches the download task for a given bounding box. 102 * 103 * @param reader the reader used to parse OSM data (see {@link OsmServerReader#parseOsm}) 104 * @param newLayer newLayer true, if the data is to be downloaded into a new layer. If false, the task 105 * selects one of the existing layers as download layer, preferably the active layer. 106 * @param downloadArea the area to download 107 * @param progressMonitor the progressMonitor 108 * @return the future representing the asynchronous task 109 * @deprecated Use {@link #download(OsmServerReader, DownloadParams, Bounds, ProgressMonitor)} 110 */ 111 @Deprecated 112 public Future<?> download(OsmServerReader reader, boolean newLayer, Bounds downloadArea, ProgressMonitor progressMonitor) { 113 return download(new DownloadTask(new DownloadParams().withNewLayer(newLayer), reader, progressMonitor, zoomAfterDownload), 114 downloadArea); 115 } 116 117 /** 118 * Asynchronously launches the download task for a given bounding box. 119 * 120 * Set <code>progressMonitor</code> to null, if the task should create, open, and close a progress monitor. 121 * Set progressMonitor to {@link NullProgressMonitor#INSTANCE} if progress information is to 122 * be discarded. 123 * 124 * You can wait for the asynchronous download task to finish by synchronizing on the returned 125 * {@link Future}, but make sure not to freeze up JOSM. Example: 126 * <pre> 127 * Future<?> future = task.download(...); 128 * // DON'T run this on the Swing EDT or JOSM will freeze 129 * future.get(); // waits for the dowload task to complete 130 * </pre> 131 * 132 * The following example uses a pattern which is better suited if a task is launched from 133 * the Swing EDT: 134 * <pre> 135 * final Future<?> future = task.download(...); 136 * Runnable runAfterTask = new Runnable() { 137 * public void run() { 138 * // this is not strictly necessary because of the type of executor service 139 * // Main.worker is initialized with, but it doesn't harm either 140 * // 141 * future.get(); // wait for the download task to complete 142 * doSomethingAfterTheTaskCompleted(); 143 * } 144 * } 145 * MainApplication.worker.submit(runAfterTask); 146 * </pre> 147 * @param reader the reader used to parse OSM data (see {@link OsmServerReader#parseOsm}) 148 * @param settings download settings 149 * @param downloadArea the area to download 150 * @param progressMonitor the progressMonitor 151 * @return the future representing the asynchronous task 152 * @since 13927 153 */ 154 public Future<?> download(OsmServerReader reader, DownloadParams settings, Bounds downloadArea, ProgressMonitor progressMonitor) { 155 return download(new DownloadTask(settings, reader, progressMonitor, zoomAfterDownload), downloadArea); 156 } 157 158 protected Future<?> download(DownloadTask downloadTask, Bounds downloadArea) { 159 this.downloadTask = downloadTask; 160 this.currentBounds = new Bounds(downloadArea); 161 // We need submit instead of execute so we can wait for it to finish and get the error 162 // message if necessary. If no one calls getErrorMessage() it just behaves like execute. 163 return MainApplication.worker.submit(downloadTask); 164 } 165 166 /** 167 * This allows subclasses to perform operations on the URL before {@link #loadUrl} is performed. 168 * @param url the original URL 169 * @return the modified URL 170 */ 171 protected String modifyUrlBeforeLoad(String url) { 172 return url; 173 } 174 175 /** 176 * Loads a given URL from the OSM Server 177 * @param settings download settings 178 * @param url The URL as String 179 */ 180 @Override 181 public Future<?> loadUrl(DownloadParams settings, String url, ProgressMonitor progressMonitor) { 182 String newUrl = modifyUrlBeforeLoad(url); 183 downloadTask = new DownloadTask(settings, 184 new OsmServerLocationReader(newUrl), 185 progressMonitor); 186 currentBounds = null; 187 // Extract .osm filename from URL to set the new layer name 188 extractOsmFilename(settings, "https?://.*/(.*\\.osm)", newUrl); 189 return MainApplication.worker.submit(downloadTask); 190 } 191 192 protected final void extractOsmFilename(DownloadParams settings, String pattern, String url) { 193 newLayerName = settings.getLayerName(); 194 if (newLayerName == null || newLayerName.isEmpty()) { 195 Matcher matcher = Pattern.compile(pattern).matcher(url); 196 newLayerName = matcher.matches() ? matcher.group(1) : null; 197 } 198 } 199 200 @Override 201 public void cancel() { 202 if (downloadTask != null) { 203 downloadTask.cancel(); 204 } 205 } 206 207 @Override 208 public boolean isSafeForRemotecontrolRequests() { 209 return true; 210 } 211 212 @Override 213 public ProjectionBounds getDownloadProjectionBounds() { 214 return downloadTask != null ? downloadTask.computeBbox(currentBounds) : null; 215 } 216 217 /** 218 * Superclass of internal download task. 219 * @since 7636 220 */ 221 public abstract static class AbstractInternalTask extends PleaseWaitRunnable { 222 223 protected final DownloadParams settings; 224 protected final boolean zoomAfterDownload; 225 protected DataSet dataSet; 226 227 /** 228 * Constructs a new {@code AbstractInternalTask}. 229 * @param settings download settings 230 * @param title message for the user 231 * @param ignoreException If true, exception will be propagated to calling code. If false then 232 * exception will be thrown directly in EDT. When this runnable is executed using executor framework 233 * then use false unless you read result of task (because exception will get lost if you don't) 234 * @param zoomAfterDownload If true, the map view will zoom to download area after download 235 */ 236 public AbstractInternalTask(DownloadParams settings, String title, boolean ignoreException, boolean zoomAfterDownload) { 237 super(title, ignoreException); 238 this.settings = Objects.requireNonNull(settings); 239 this.zoomAfterDownload = zoomAfterDownload; 240 } 241 242 /** 243 * Constructs a new {@code AbstractInternalTask}. 244 * @param settings download settings 245 * @param title message for the user 246 * @param progressMonitor progress monitor 247 * @param ignoreException If true, exception will be propagated to calling code. If false then 248 * exception will be thrown directly in EDT. When this runnable is executed using executor framework 249 * then use false unless you read result of task (because exception will get lost if you don't) 250 * @param zoomAfterDownload If true, the map view will zoom to download area after download 251 */ 252 public AbstractInternalTask(DownloadParams settings, String title, ProgressMonitor progressMonitor, boolean ignoreException, 253 boolean zoomAfterDownload) { 254 super(title, progressMonitor, ignoreException); 255 this.settings = Objects.requireNonNull(settings); 256 this.zoomAfterDownload = zoomAfterDownload; 257 } 258 259 protected OsmDataLayer getEditLayer() { 260 return MainApplication.getLayerManager().getEditLayer(); 261 } 262 263 /** 264 * Returns the number of modifiable data layers 265 * @return number of modifiable data layers 266 * @deprecated Use {@link #getNumModifiableDataLayers} 267 */ 268 @Deprecated 269 protected int getNumDataLayers() { 270 return (int) getNumModifiableDataLayers(); 271 } 272 273 private static Stream<OsmDataLayer> getModifiableDataLayers() { 274 return MainApplication.getLayerManager().getLayersOfType(OsmDataLayer.class) 275 .stream().filter(OsmDataLayer::isDownloadable); 276 } 277 278 /** 279 * Returns the number of modifiable data layers 280 * @return number of modifiable data layers 281 * @since 13434 282 */ 283 protected long getNumModifiableDataLayers() { 284 return getModifiableDataLayers().count(); 285 } 286 287 /** 288 * Returns the first modifiable data layer 289 * @return the first modifiable data layer 290 * @since 13434 291 */ 292 protected OsmDataLayer getFirstModifiableDataLayer() { 293 return getModifiableDataLayers().findFirst().orElse(null); 294 } 295 296 protected OsmDataLayer createNewLayer(String layerName) { 297 if (layerName == null || layerName.isEmpty()) { 298 layerName = settings.getLayerName(); 299 } 300 if (layerName == null || layerName.isEmpty()) { 301 layerName = OsmDataLayer.createNewName(); 302 } 303 if (settings.getDownloadPolicy() != null) { 304 dataSet.setDownloadPolicy(settings.getDownloadPolicy()); 305 } 306 if (settings.getUploadPolicy() != null) { 307 dataSet.setUploadPolicy(settings.getUploadPolicy()); 308 } 309 if (dataSet.isLocked() && !settings.isLocked()) { 310 dataSet.unlock(); 311 } else if (!dataSet.isLocked() && settings.isLocked()) { 312 dataSet.lock(); 313 } 314 return new OsmDataLayer(dataSet, layerName, null); 315 } 316 317 protected OsmDataLayer createNewLayer() { 318 return createNewLayer(null); 319 } 320 321 protected ProjectionBounds computeBbox(Bounds bounds) { 322 BoundingXYVisitor v = new BoundingXYVisitor(); 323 if (bounds != null) { 324 v.visit(bounds); 325 } else { 326 v.computeBoundingBox(dataSet.getNodes()); 327 } 328 return v.getBounds(); 329 } 330 331 protected OsmDataLayer addNewLayerIfRequired(String newLayerName) { 332 long numDataLayers = getNumModifiableDataLayers(); 333 if (settings.isNewLayer() || numDataLayers == 0 || (numDataLayers > 1 && getEditLayer() == null)) { 334 // the user explicitly wants a new layer, we don't have any layer at all 335 // or it is not clear which layer to merge to 336 final OsmDataLayer layer = createNewLayer(newLayerName); 337 MainApplication.getLayerManager().addLayer(layer, zoomAfterDownload); 338 return layer; 339 } 340 return null; 341 } 342 343 protected void loadData(String newLayerName, Bounds bounds) { 344 OsmDataLayer layer = addNewLayerIfRequired(newLayerName); 345 if (layer == null) { 346 layer = getEditLayer(); 347 if (layer == null || !layer.isDownloadable()) { 348 layer = getFirstModifiableDataLayer(); 349 } 350 Collection<OsmPrimitive> primitivesToUpdate = searchPrimitivesToUpdate(bounds, layer.getDataSet()); 351 layer.mergeFrom(dataSet); 352 MapFrame map = MainApplication.getMap(); 353 if (map != null && zoomAfterDownload && bounds != null) { 354 map.mapView.zoomTo(new ViewportData(computeBbox(bounds))); 355 } 356 if (!primitivesToUpdate.isEmpty()) { 357 MainApplication.worker.submit(new UpdatePrimitivesTask(layer, primitivesToUpdate)); 358 } 359 layer.onPostDownloadFromServer(); 360 } 361 } 362 363 /** 364 * Look for primitives deleted on server (thus absent from downloaded data) 365 * but still present in existing data layer 366 * @param bounds download bounds 367 * @param ds existing data set 368 * @return the primitives to update 369 */ 370 private Collection<OsmPrimitive> searchPrimitivesToUpdate(Bounds bounds, DataSet ds) { 371 if (bounds == null) 372 return Collections.emptySet(); 373 Collection<OsmPrimitive> col = new ArrayList<>(); 374 ds.searchNodes(bounds.toBBox()).stream().filter(n -> !n.isNew() && !dataSet.containsNode(n)).forEachOrdered(col::add); 375 if (!col.isEmpty()) { 376 Set<Way> ways = new HashSet<>(); 377 Set<Relation> rels = new HashSet<>(); 378 for (OsmPrimitive n : col) { 379 for (OsmPrimitive ref : n.getReferrers()) { 380 if (ref.isNew()) { 381 continue; 382 } else if (ref instanceof Way) { 383 ways.add((Way) ref); 384 } else if (ref instanceof Relation) { 385 rels.add((Relation) ref); 386 } 387 } 388 } 389 ways.stream().filter(w -> !dataSet.containsWay(w)).forEachOrdered(col::add); 390 rels.stream().filter(r -> !dataSet.containsRelation(r)).forEachOrdered(col::add); 391 } 392 return col; 393 } 394 } 395 396 protected class DownloadTask extends AbstractInternalTask { 397 protected final OsmServerReader reader; 398 399 /** 400 * Constructs a new {@code DownloadTask}. 401 * @param newLayer if {@code true}, force download to a new layer 402 * @param reader OSM data reader 403 * @param progressMonitor progress monitor 404 * @deprecated Use {@code DownloadTask(DownloadParams, OsmServerReader, ProgressMonitor)} 405 */ 406 @Deprecated 407 public DownloadTask(boolean newLayer, OsmServerReader reader, ProgressMonitor progressMonitor) { 408 this(new DownloadParams().withNewLayer(newLayer), reader, progressMonitor); 409 } 410 411 /** 412 * Constructs a new {@code DownloadTask}. 413 * @param newLayer if {@code true}, force download to a new layer 414 * @param reader OSM data reader 415 * @param progressMonitor progress monitor 416 * @param zoomAfterDownload If true, the map view will zoom to download area after download 417 * @deprecated Use {@code DownloadTask(DownloadParams, OsmServerReader, ProgressMonitor, boolean)} 418 */ 419 @Deprecated 420 public DownloadTask(boolean newLayer, OsmServerReader reader, ProgressMonitor progressMonitor, boolean zoomAfterDownload) { 421 this(new DownloadParams().withNewLayer(newLayer), reader, progressMonitor, zoomAfterDownload); 422 } 423 424 /** 425 * Constructs a new {@code DownloadTask}. 426 * @param settings download settings 427 * @param reader OSM data reader 428 * @param progressMonitor progress monitor 429 * @since 13927 430 */ 431 public DownloadTask(DownloadParams settings, OsmServerReader reader, ProgressMonitor progressMonitor) { 432 this(settings, reader, progressMonitor, true); 433 } 434 435 /** 436 * Constructs a new {@code DownloadTask}. 437 * @param settings download settings 438 * @param reader OSM data reader 439 * @param progressMonitor progress monitor 440 * @param zoomAfterDownload If true, the map view will zoom to download area after download 441 * @since 13927 442 */ 443 public DownloadTask(DownloadParams settings, OsmServerReader reader, ProgressMonitor progressMonitor, boolean zoomAfterDownload) { 444 super(settings, tr("Downloading data"), progressMonitor, false, zoomAfterDownload); 445 this.reader = reader; 446 } 447 448 protected DataSet parseDataSet() throws OsmTransferException { 449 return reader.parseOsm(progressMonitor.createSubTaskMonitor(ProgressMonitor.ALL_TICKS, false)); 450 } 451 452 @Override 453 public void realRun() throws IOException, SAXException, OsmTransferException { 454 try { 455 if (isCanceled()) 456 return; 457 dataSet = parseDataSet(); 458 } catch (OsmTransferException e) { 459 if (isCanceled()) { 460 Logging.info(tr("Ignoring exception because download has been canceled. Exception was: {0}", e.toString())); 461 return; 462 } 463 if (e instanceof OsmTransferCanceledException) { 464 setCanceled(true); 465 return; 466 } else { 467 rememberException(e); 468 } 469 DownloadOsmTask.this.setFailed(true); 470 } 471 } 472 473 @Override 474 protected void finish() { 475 if (isFailed() || isCanceled()) 476 return; 477 if (dataSet == null) 478 return; // user canceled download or error occurred 479 if (dataSet.allPrimitives().isEmpty()) { 480 if (warnAboutEmptyArea) { 481 rememberErrorMessage(tr("No data found in this area.")); 482 } 483 // need to synthesize a download bounds lest the visual indication of downloaded area doesn't work 484 dataSet.addDataSource(new DataSource(currentBounds != null ? currentBounds : 485 new Bounds(LatLon.ZERO), "OpenStreetMap server")); 486 } 487 488 rememberDownloadedData(dataSet); 489 loadData(newLayerName, currentBounds); 490 } 491 492 @Override 493 protected void cancel() { 494 setCanceled(true); 495 if (reader != null) { 496 reader.cancel(); 497 } 498 } 499 } 500 501 @Override 502 public String getConfirmationMessage(URL url) { 503 if (url != null) { 504 String urlString = url.toExternalForm(); 505 if (urlString.matches(OsmUrlPattern.OSM_API_URL.pattern())) { 506 // TODO: proper i18n after stabilization 507 Collection<String> items = new ArrayList<>(); 508 items.add(tr("OSM Server URL:") + ' ' + url.getHost()); 509 items.add(tr("Command")+": "+url.getPath()); 510 if (url.getQuery() != null) { 511 items.add(tr("Request details: {0}", url.getQuery().replaceAll(",\\s*", ", "))); 512 } 513 return Utils.joinAsHtmlUnorderedList(items); 514 } 515 // TODO: other APIs 516 } 517 return null; 518 } 519}