001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.io; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.io.IOException; 007import java.io.InputStream; 008import java.net.Authenticator.RequestorType; 009import java.net.HttpURLConnection; 010import java.net.MalformedURLException; 011import java.net.URL; 012import java.util.List; 013 014import javax.xml.parsers.ParserConfigurationException; 015 016import org.openstreetmap.josm.Main; 017import org.openstreetmap.josm.data.gpx.GpxData; 018import org.openstreetmap.josm.data.notes.Note; 019import org.openstreetmap.josm.data.osm.DataSet; 020import org.openstreetmap.josm.gui.progress.ProgressMonitor; 021import org.openstreetmap.josm.io.auth.CredentialsAgentException; 022import org.openstreetmap.josm.io.auth.CredentialsManager; 023import org.openstreetmap.josm.tools.HttpClient; 024import org.openstreetmap.josm.tools.Logging; 025import org.openstreetmap.josm.tools.Utils; 026import org.openstreetmap.josm.tools.XmlParsingException; 027import org.w3c.dom.Document; 028import org.w3c.dom.Node; 029import org.xml.sax.SAXException; 030 031/** 032 * This DataReader reads directly from the REST API of the osm server. 033 * 034 * It supports plain text transfer as well as gzip or deflate encoded transfers; 035 * if compressed transfers are unwanted, set property osm-server.use-compression 036 * to false. 037 * 038 * @author imi 039 */ 040public abstract class OsmServerReader extends OsmConnection { 041 private final OsmApi api = OsmApi.getOsmApi(); 042 private boolean doAuthenticate; 043 protected boolean gpxParsedProperly; 044 045 /** 046 * Constructs a new {@code OsmServerReader}. 047 */ 048 public OsmServerReader() { 049 try { 050 doAuthenticate = OsmApi.isUsingOAuth() && CredentialsManager.getInstance().lookupOAuthAccessToken() != null; 051 } catch (CredentialsAgentException e) { 052 Logging.warn(e); 053 } 054 } 055 056 /** 057 * Open a connection to the given url and return a reader on the input stream 058 * from that connection. In case of user cancel, return <code>null</code>. 059 * Relative URL's are directed to API base URL. 060 * @param urlStr The url to connect to. 061 * @param progressMonitor progress monitoring and abort handler 062 * @return A reader reading the input stream (servers answer) or <code>null</code>. 063 * @throws OsmTransferException if data transfer errors occur 064 */ 065 protected InputStream getInputStream(String urlStr, ProgressMonitor progressMonitor) throws OsmTransferException { 066 return getInputStream(urlStr, progressMonitor, null); 067 } 068 069 /** 070 * Open a connection to the given url and return a reader on the input stream 071 * from that connection. In case of user cancel, return <code>null</code>. 072 * Relative URL's are directed to API base URL. 073 * @param urlStr The url to connect to. 074 * @param progressMonitor progress monitoring and abort handler 075 * @param reason The reason to show on console. Can be {@code null} if no reason is given 076 * @return A reader reading the input stream (servers answer) or <code>null</code>. 077 * @throws OsmTransferException if data transfer errors occur 078 */ 079 protected InputStream getInputStream(String urlStr, ProgressMonitor progressMonitor, String reason) throws OsmTransferException { 080 try { 081 api.initialize(progressMonitor); 082 String url = urlStr.startsWith("http") ? urlStr : (getBaseUrl() + urlStr); 083 return getInputStreamRaw(url, progressMonitor, reason); 084 } finally { 085 progressMonitor.invalidate(); 086 } 087 } 088 089 /** 090 * Return the base URL for relative URL requests 091 * @return base url of API 092 */ 093 protected String getBaseUrl() { 094 return api.getBaseUrl(); 095 } 096 097 /** 098 * Open a connection to the given url and return a reader on the input stream 099 * from that connection. In case of user cancel, return <code>null</code>. 100 * @param urlStr The exact url to connect to. 101 * @param progressMonitor progress monitoring and abort handler 102 * @return An reader reading the input stream (servers answer) or <code>null</code>. 103 * @throws OsmTransferException if data transfer errors occur 104 */ 105 protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor) throws OsmTransferException { 106 return getInputStreamRaw(urlStr, progressMonitor, null); 107 } 108 109 /** 110 * Open a connection to the given url and return a reader on the input stream 111 * from that connection. In case of user cancel, return <code>null</code>. 112 * @param urlStr The exact url to connect to. 113 * @param progressMonitor progress monitoring and abort handler 114 * @param reason The reason to show on console. Can be {@code null} if no reason is given 115 * @return An reader reading the input stream (servers answer) or <code>null</code>. 116 * @throws OsmTransferException if data transfer errors occur 117 */ 118 protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason) throws OsmTransferException { 119 return getInputStreamRaw(urlStr, progressMonitor, reason, false); 120 } 121 122 /** 123 * Open a connection to the given url (if HTTP, trough a GET request) and return a reader on the input stream 124 * from that connection. In case of user cancel, return <code>null</code>. 125 * @param urlStr The exact url to connect to. 126 * @param progressMonitor progress monitoring and abort handler 127 * @param reason The reason to show on console. Can be {@code null} if no reason is given 128 * @param uncompressAccordingToContentDisposition Whether to inspect the HTTP header {@code Content-Disposition} 129 * for {@code filename} and uncompress a gzip/bzip2/xz/zip stream. 130 * @return An reader reading the input stream (servers answer) or <code>null</code>. 131 * @throws OsmTransferException if data transfer errors occur 132 */ 133 protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason, 134 boolean uncompressAccordingToContentDisposition) throws OsmTransferException { 135 return getInputStreamRaw(urlStr, progressMonitor, reason, uncompressAccordingToContentDisposition, "GET", null); 136 } 137 138 /** 139 * Open a connection to the given url (if HTTP, with the specified method) and return a reader on the input stream 140 * from that connection. In case of user cancel, return <code>null</code>. 141 * @param urlStr The exact url to connect to. 142 * @param progressMonitor progress monitoring and abort handler 143 * @param reason The reason to show on console. Can be {@code null} if no reason is given 144 * @param uncompressAccordingToContentDisposition Whether to inspect the HTTP header {@code Content-Disposition} 145 * for {@code filename} and uncompress a gzip/bzip2/xz/zip stream. 146 * @param httpMethod HTTP method ("GET", "POST" or "PUT") 147 * @param requestBody HTTP request body (for "POST" and "PUT" methods only). Must be null for "GET" method. 148 * @return An reader reading the input stream (servers answer) or <code>null</code>. 149 * @throws OsmTransferException if data transfer errors occur 150 * @since 12596 151 */ 152 @SuppressWarnings("resource") 153 protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason, 154 boolean uncompressAccordingToContentDisposition, String httpMethod, byte[] requestBody) throws OsmTransferException { 155 try { 156 OnlineResource.JOSM_WEBSITE.checkOfflineAccess(urlStr, Main.getJOSMWebsite()); 157 OnlineResource.OSM_API.checkOfflineAccess(urlStr, OsmApi.getOsmApi().getServerUrl()); 158 159 URL url = null; 160 try { 161 url = new URL(urlStr.replace(" ", "%20")); 162 } catch (MalformedURLException e) { 163 throw new OsmTransferException(e); 164 } 165 166 String protocol = url.getProtocol(); 167 if ("file".equals(protocol) || "jar".equals(protocol)) { 168 try { 169 return Utils.openStream(url); 170 } catch (IOException e) { 171 throw new OsmTransferException(e); 172 } 173 } 174 175 final HttpClient client = HttpClient.create(url, httpMethod) 176 .setFinishOnCloseOutput(false) 177 .setReasonForRequest(reason) 178 .setOutputMessage(tr("Downloading data...")) 179 .setRequestBody(requestBody); 180 activeConnection = client; 181 adaptRequest(client); 182 if (doAuthenticate) { 183 addAuth(client); 184 } 185 if (cancel) 186 throw new OsmTransferCanceledException("Operation canceled"); 187 188 final HttpClient.Response response; 189 try { 190 response = client.connect(progressMonitor); 191 } catch (IOException e) { 192 Logging.error(e); 193 OsmTransferException ote = new OsmTransferException( 194 tr("Could not connect to the OSM server. Please check your internet connection."), e); 195 ote.setUrl(url.toString()); 196 throw ote; 197 } 198 try { 199 if (response.getResponseCode() == HttpURLConnection.HTTP_UNAUTHORIZED) { 200 CredentialsManager.getInstance().purgeCredentialsCache(RequestorType.SERVER); 201 throw new OsmApiException(HttpURLConnection.HTTP_UNAUTHORIZED, null, null); 202 } 203 204 if (response.getResponseCode() == HttpURLConnection.HTTP_PROXY_AUTH) 205 throw new OsmTransferCanceledException("Proxy Authentication Required"); 206 207 if (response.getResponseCode() != HttpURLConnection.HTTP_OK) { 208 String errorHeader = response.getHeaderField("Error"); 209 String errorBody = fetchResponseText(response); 210 throw new OsmApiException(response.getResponseCode(), errorHeader, errorBody, url.toString(), null, 211 response.getContentType()); 212 } 213 214 response.uncompressAccordingToContentDisposition(uncompressAccordingToContentDisposition); 215 return response.getContent(); 216 } catch (OsmTransferException e) { 217 throw e; 218 } catch (IOException e) { 219 throw new OsmTransferException(e); 220 } 221 } finally { 222 progressMonitor.invalidate(); 223 } 224 } 225 226 private static String fetchResponseText(final HttpClient.Response response) { 227 try { 228 return response.fetchContent(); 229 } catch (IOException e) { 230 Logging.error(e); 231 return tr("Reading error text failed."); 232 } 233 } 234 235 /** 236 * Allows subclasses to modify the request. 237 * @param request the prepared request 238 * @since 9308 239 */ 240 protected void adaptRequest(HttpClient request) { 241 } 242 243 /** 244 * Download OSM files from somewhere 245 * @param progressMonitor The progress monitor 246 * @return The corresponding dataset 247 * @throws OsmTransferException if any error occurs 248 */ 249 public abstract DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException; 250 251 /** 252 * Download compressed OSM files from somewhere 253 * @param progressMonitor The progress monitor 254 * @param compression compression to use 255 * @return The corresponding dataset 256 * @throws OsmTransferException if any error occurs 257 * @since 13352 258 */ 259 public DataSet parseOsm(ProgressMonitor progressMonitor, Compression compression) throws OsmTransferException { 260 return null; 261 } 262 263 /** 264 * Download OSM Change uncompressed files from somewhere 265 * @param progressMonitor The progress monitor 266 * @return The corresponding dataset 267 * @throws OsmTransferException if any error occurs 268 */ 269 public DataSet parseOsmChange(ProgressMonitor progressMonitor) throws OsmTransferException { 270 return null; 271 } 272 273 /** 274 * Download OSM Change compressed files from somewhere 275 * @param progressMonitor The progress monitor 276 * @param compression compression to use 277 * @return The corresponding dataset 278 * @throws OsmTransferException if any error occurs 279 * @since 13352 280 */ 281 public DataSet parseOsmChange(ProgressMonitor progressMonitor, Compression compression) throws OsmTransferException { 282 return null; 283 } 284 285 /** 286 * Download BZip2-compressed OSM Change files from somewhere 287 * @param progressMonitor The progress monitor 288 * @return The corresponding dataset 289 * @throws OsmTransferException if any error occurs 290 * @deprecated use {@link #parseOsmChange(ProgressMonitor, Compression)} instead 291 */ 292 @Deprecated 293 public DataSet parseOsmChangeBzip2(ProgressMonitor progressMonitor) throws OsmTransferException { 294 return parseOsmChange(progressMonitor, Compression.BZIP2); 295 } 296 297 /** 298 * Download GZip-compressed OSM Change files from somewhere 299 * @param progressMonitor The progress monitor 300 * @return The corresponding dataset 301 * @throws OsmTransferException if any error occurs 302 * @deprecated use {@link #parseOsmChange(ProgressMonitor, Compression)} instead 303 */ 304 @Deprecated 305 public DataSet parseOsmChangeGzip(ProgressMonitor progressMonitor) throws OsmTransferException { 306 return parseOsmChange(progressMonitor, Compression.GZIP); 307 } 308 309 /** 310 * Retrieve raw gps waypoints from the server API. 311 * @param progressMonitor The progress monitor 312 * @return The corresponding GPX tracks 313 * @throws OsmTransferException if any error occurs 314 */ 315 public GpxData parseRawGps(ProgressMonitor progressMonitor) throws OsmTransferException { 316 return null; 317 } 318 319 /** 320 * Retrieve compressed GPX files from somewhere. 321 * @param progressMonitor The progress monitor 322 * @param compression compression to use 323 * @return The corresponding GPX tracks 324 * @throws OsmTransferException if any error occurs 325 * @since 13352 326 */ 327 public GpxData parseRawGps(ProgressMonitor progressMonitor, Compression compression) throws OsmTransferException { 328 return null; 329 } 330 331 /** 332 * Retrieve BZip2-compressed GPX files from somewhere. 333 * @param progressMonitor The progress monitor 334 * @return The corresponding GPX tracks 335 * @throws OsmTransferException if any error occurs 336 * @deprecated use {@link #parseRawGps(ProgressMonitor, Compression)} instead 337 * @since 6244 338 */ 339 @Deprecated 340 public GpxData parseRawGpsBzip2(ProgressMonitor progressMonitor) throws OsmTransferException { 341 return parseRawGps(progressMonitor, Compression.BZIP2); 342 } 343 344 /** 345 * Download BZip2-compressed OSM files from somewhere 346 * @param progressMonitor The progress monitor 347 * @return The corresponding dataset 348 * @throws OsmTransferException if any error occurs 349 * @deprecated use {@link #parseOsm(ProgressMonitor, Compression)} instead 350 */ 351 @Deprecated 352 public DataSet parseOsmBzip2(ProgressMonitor progressMonitor) throws OsmTransferException { 353 return parseOsm(progressMonitor, Compression.BZIP2); 354 } 355 356 /** 357 * Download GZip-compressed OSM files from somewhere 358 * @param progressMonitor The progress monitor 359 * @return The corresponding dataset 360 * @throws OsmTransferException if any error occurs 361 * @deprecated use {@link #parseOsm(ProgressMonitor, Compression)} instead 362 */ 363 @Deprecated 364 public DataSet parseOsmGzip(ProgressMonitor progressMonitor) throws OsmTransferException { 365 return parseOsm(progressMonitor, Compression.GZIP); 366 } 367 368 /** 369 * Download Zip-compressed OSM files from somewhere 370 * @param progressMonitor The progress monitor 371 * @return The corresponding dataset 372 * @throws OsmTransferException if any error occurs 373 * @deprecated use {@link #parseOsm(ProgressMonitor, Compression)} instead 374 * @since 6882 375 */ 376 @Deprecated 377 public DataSet parseOsmZip(final ProgressMonitor progressMonitor) throws OsmTransferException { 378 return parseOsm(progressMonitor, Compression.ZIP); 379 } 380 381 /** 382 * Returns true if this reader is adding authentication credentials to the read 383 * request sent to the server. 384 * 385 * @return true if this reader is adding authentication credentials to the read 386 * request sent to the server 387 */ 388 public boolean isDoAuthenticate() { 389 return doAuthenticate; 390 } 391 392 /** 393 * Sets whether this reader adds authentication credentials to the read 394 * request sent to the server. 395 * 396 * @param doAuthenticate true if this reader adds authentication credentials to the read 397 * request sent to the server 398 */ 399 public void setDoAuthenticate(boolean doAuthenticate) { 400 this.doAuthenticate = doAuthenticate; 401 } 402 403 /** 404 * Determines if the GPX data has been parsed properly. 405 * @return true if the GPX data has been parsed properly, false otherwise 406 * @see GpxReader#parse 407 */ 408 public final boolean isGpxParsedProperly() { 409 return gpxParsedProperly; 410 } 411 412 /** 413 * Downloads notes from the API, given API limit parameters 414 * 415 * @param noteLimit How many notes to download. 416 * @param daysClosed Return notes closed this many days in the past. -1 means all notes, ever. 0 means only unresolved notes. 417 * @param progressMonitor Progress monitor for user feedback 418 * @return List of notes returned by the API 419 * @throws OsmTransferException if any errors happen 420 */ 421 public List<Note> parseNotes(int noteLimit, int daysClosed, ProgressMonitor progressMonitor) throws OsmTransferException { 422 return null; 423 } 424 425 /** 426 * Downloads notes from a given raw URL. The URL is assumed to be complete and no API limits are added 427 * 428 * @param progressMonitor progress monitor 429 * @return A list of notes parsed from the URL 430 * @throws OsmTransferException if any error occurs during dialog with OSM API 431 */ 432 public List<Note> parseRawNotes(final ProgressMonitor progressMonitor) throws OsmTransferException { 433 return null; 434 } 435 436 /** 437 * Download notes from a URL that contains a compressed notes dump file 438 * @param progressMonitor progress monitor 439 * @param compression compression to use 440 * @return A list of notes parsed from the URL 441 * @throws OsmTransferException if any error occurs during dialog with OSM API 442 * @since 13352 443 */ 444 public List<Note> parseRawNotes(ProgressMonitor progressMonitor, Compression compression) throws OsmTransferException { 445 return null; 446 } 447 448 /** 449 * Download notes from a URL that contains a bzip2 compressed notes dump file 450 * @param progressMonitor progress monitor 451 * @return A list of notes parsed from the URL 452 * @throws OsmTransferException if any error occurs during dialog with OSM API 453 * @deprecated Use {@link #parseRawNotes(ProgressMonitor, Compression)} instead 454 */ 455 @Deprecated 456 public List<Note> parseRawNotesBzip2(final ProgressMonitor progressMonitor) throws OsmTransferException { 457 return parseRawNotes(progressMonitor, Compression.BZIP2); 458 } 459 460 /** 461 * Returns an attribute from the given DOM node. 462 * @param node DOM node 463 * @param name attribute name 464 * @return attribute value for the given attribute 465 * @since 12510 466 */ 467 protected static String getAttribute(Node node, String name) { 468 return node.getAttributes().getNamedItem(name).getNodeValue(); 469 } 470 471 /** 472 * DOM document parser. 473 * @param <R> resulting type 474 * @since 12510 475 */ 476 @FunctionalInterface 477 protected interface DomParser<R> { 478 /** 479 * Parses a given DOM document. 480 * @param doc DOM document 481 * @return parsed data 482 * @throws XmlParsingException if an XML parsing error occurs 483 */ 484 R parse(Document doc) throws XmlParsingException; 485 } 486 487 /** 488 * Fetches generic data from the DOM document resulting an API call. 489 * @param api the OSM API call 490 * @param subtask the subtask translated message 491 * @param parser the parser converting the DOM document (OSM API result) 492 * @param <T> data type 493 * @param monitor The progress monitor 494 * @param reason The reason to show on console. Can be {@code null} if no reason is given 495 * @return The converted data 496 * @throws OsmTransferException if something goes wrong 497 * @since 12510 498 */ 499 public <T> T fetchData(String api, String subtask, DomParser<T> parser, ProgressMonitor monitor, String reason) 500 throws OsmTransferException { 501 try { 502 monitor.beginTask(""); 503 monitor.indeterminateSubTask(subtask); 504 try (InputStream in = getInputStream(api, monitor.createSubTaskMonitor(1, true), reason)) { 505 return parser.parse(Utils.parseSafeDOM(in)); 506 } 507 } catch (OsmTransferException e) { 508 throw e; 509 } catch (IOException | ParserConfigurationException | SAXException e) { 510 throw new OsmTransferException(e); 511 } finally { 512 monitor.finishTask(); 513 } 514 } 515}