001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.io; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.io.IOException; 007import java.io.InputStream; 008import java.net.Authenticator.RequestorType; 009import java.net.HttpURLConnection; 010import java.net.MalformedURLException; 011import java.net.URL; 012import java.util.List; 013 014import javax.xml.parsers.ParserConfigurationException; 015 016import org.openstreetmap.josm.Main; 017import org.openstreetmap.josm.data.gpx.GpxData; 018import org.openstreetmap.josm.data.notes.Note; 019import org.openstreetmap.josm.data.osm.DataSet; 020import org.openstreetmap.josm.gui.progress.ProgressMonitor; 021import org.openstreetmap.josm.io.auth.CredentialsAgentException; 022import org.openstreetmap.josm.io.auth.CredentialsManager; 023import org.openstreetmap.josm.tools.HttpClient; 024import org.openstreetmap.josm.tools.Logging; 025import org.openstreetmap.josm.tools.Utils; 026import org.openstreetmap.josm.tools.XmlParsingException; 027import org.openstreetmap.josm.tools.XmlUtils; 028import org.w3c.dom.Document; 029import org.w3c.dom.Node; 030import org.xml.sax.SAXException; 031 032/** 033 * This DataReader reads directly from the REST API of the osm server. 034 * 035 * It supports plain text transfer as well as gzip or deflate encoded transfers; 036 * if compressed transfers are unwanted, set property osm-server.use-compression 037 * to false. 038 * 039 * @author imi 040 */ 041public abstract class OsmServerReader extends OsmConnection { 042 private final OsmApi api = OsmApi.getOsmApi(); 043 private boolean doAuthenticate; 044 protected boolean gpxParsedProperly; 045 046 /** 047 * Constructs a new {@code OsmServerReader}. 048 */ 049 public OsmServerReader() { 050 try { 051 doAuthenticate = OsmApi.isUsingOAuth() && CredentialsManager.getInstance().lookupOAuthAccessToken() != null; 052 } catch (CredentialsAgentException e) { 053 Logging.warn(e); 054 } 055 } 056 057 /** 058 * Open a connection to the given url and return a reader on the input stream 059 * from that connection. In case of user cancel, return <code>null</code>. 060 * Relative URL's are directed to API base URL. 061 * @param urlStr The url to connect to. 062 * @param progressMonitor progress monitoring and abort handler 063 * @return A reader reading the input stream (servers answer) or <code>null</code>. 064 * @throws OsmTransferException if data transfer errors occur 065 */ 066 protected InputStream getInputStream(String urlStr, ProgressMonitor progressMonitor) throws OsmTransferException { 067 return getInputStream(urlStr, progressMonitor, null); 068 } 069 070 /** 071 * Open a connection to the given url and return a reader on the input stream 072 * from that connection. In case of user cancel, return <code>null</code>. 073 * Relative URL's are directed to API base URL. 074 * @param urlStr The url to connect to. 075 * @param progressMonitor progress monitoring and abort handler 076 * @param reason The reason to show on console. Can be {@code null} if no reason is given 077 * @return A reader reading the input stream (servers answer) or <code>null</code>. 078 * @throws OsmTransferException if data transfer errors occur 079 */ 080 protected InputStream getInputStream(String urlStr, ProgressMonitor progressMonitor, String reason) throws OsmTransferException { 081 try { 082 api.initialize(progressMonitor); 083 String url = urlStr.startsWith("http") ? urlStr : (getBaseUrl() + urlStr); 084 return getInputStreamRaw(url, progressMonitor, reason); 085 } finally { 086 progressMonitor.invalidate(); 087 } 088 } 089 090 /** 091 * Return the base URL for relative URL requests 092 * @return base url of API 093 */ 094 protected String getBaseUrl() { 095 return api.getBaseUrl(); 096 } 097 098 /** 099 * Open a connection to the given url and return a reader on the input stream 100 * from that connection. In case of user cancel, return <code>null</code>. 101 * @param urlStr The exact url to connect to. 102 * @param progressMonitor progress monitoring and abort handler 103 * @return An reader reading the input stream (servers answer) or <code>null</code>. 104 * @throws OsmTransferException if data transfer errors occur 105 */ 106 protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor) throws OsmTransferException { 107 return getInputStreamRaw(urlStr, progressMonitor, null); 108 } 109 110 /** 111 * Open a connection to the given url and return a reader on the input stream 112 * from that connection. In case of user cancel, return <code>null</code>. 113 * @param urlStr The exact url to connect to. 114 * @param progressMonitor progress monitoring and abort handler 115 * @param reason The reason to show on console. Can be {@code null} if no reason is given 116 * @return An reader reading the input stream (servers answer) or <code>null</code>. 117 * @throws OsmTransferException if data transfer errors occur 118 */ 119 protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason) throws OsmTransferException { 120 return getInputStreamRaw(urlStr, progressMonitor, reason, false); 121 } 122 123 /** 124 * Open a connection to the given url (if HTTP, trough a GET request) and return a reader on the input stream 125 * from that connection. In case of user cancel, return <code>null</code>. 126 * @param urlStr The exact url to connect to. 127 * @param progressMonitor progress monitoring and abort handler 128 * @param reason The reason to show on console. Can be {@code null} if no reason is given 129 * @param uncompressAccordingToContentDisposition Whether to inspect the HTTP header {@code Content-Disposition} 130 * for {@code filename} and uncompress a gzip/bzip2/xz/zip stream. 131 * @return An reader reading the input stream (servers answer) or <code>null</code>. 132 * @throws OsmTransferException if data transfer errors occur 133 */ 134 protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason, 135 boolean uncompressAccordingToContentDisposition) throws OsmTransferException { 136 return getInputStreamRaw(urlStr, progressMonitor, reason, uncompressAccordingToContentDisposition, "GET", null); 137 } 138 139 /** 140 * Open a connection to the given url (if HTTP, with the specified method) and return a reader on the input stream 141 * from that connection. In case of user cancel, return <code>null</code>. 142 * @param urlStr The exact url to connect to. 143 * @param progressMonitor progress monitoring and abort handler 144 * @param reason The reason to show on console. Can be {@code null} if no reason is given 145 * @param uncompressAccordingToContentDisposition Whether to inspect the HTTP header {@code Content-Disposition} 146 * for {@code filename} and uncompress a gzip/bzip2/xz/zip stream. 147 * @param httpMethod HTTP method ("GET", "POST" or "PUT") 148 * @param requestBody HTTP request body (for "POST" and "PUT" methods only). Must be null for "GET" method. 149 * @return An reader reading the input stream (servers answer) or <code>null</code>. 150 * @throws OsmTransferException if data transfer errors occur 151 * @since 12596 152 */ 153 @SuppressWarnings("resource") 154 protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason, 155 boolean uncompressAccordingToContentDisposition, String httpMethod, byte[] requestBody) throws OsmTransferException { 156 try { 157 OnlineResource.JOSM_WEBSITE.checkOfflineAccess(urlStr, Main.getJOSMWebsite()); 158 OnlineResource.OSM_API.checkOfflineAccess(urlStr, OsmApi.getOsmApi().getServerUrl()); 159 160 URL url = null; 161 try { 162 url = new URL(urlStr.replace(" ", "%20")); 163 } catch (MalformedURLException e) { 164 throw new OsmTransferException(e); 165 } 166 167 String protocol = url.getProtocol(); 168 if ("file".equals(protocol) || "jar".equals(protocol)) { 169 try { 170 return Utils.openStream(url); 171 } catch (IOException e) { 172 throw new OsmTransferException(e); 173 } 174 } 175 176 final HttpClient client = HttpClient.create(url, httpMethod) 177 .setFinishOnCloseOutput(false) 178 .setReasonForRequest(reason) 179 .setOutputMessage(tr("Downloading data...")) 180 .setRequestBody(requestBody); 181 activeConnection = client; 182 adaptRequest(client); 183 if (doAuthenticate) { 184 addAuth(client); 185 } 186 if (cancel) 187 throw new OsmTransferCanceledException("Operation canceled"); 188 189 final HttpClient.Response response; 190 try { 191 response = client.connect(progressMonitor); 192 } catch (IOException e) { 193 Logging.error(e); 194 OsmTransferException ote = new OsmTransferException( 195 tr("Could not connect to the OSM server. Please check your internet connection."), e); 196 ote.setUrl(url.toString()); 197 throw ote; 198 } 199 try { 200 if (response.getResponseCode() == HttpURLConnection.HTTP_UNAUTHORIZED) { 201 CredentialsManager.getInstance().purgeCredentialsCache(RequestorType.SERVER); 202 throw new OsmApiException(HttpURLConnection.HTTP_UNAUTHORIZED, null, null); 203 } 204 205 if (response.getResponseCode() == HttpURLConnection.HTTP_PROXY_AUTH) 206 throw new OsmTransferCanceledException("Proxy Authentication Required"); 207 208 if (response.getResponseCode() != HttpURLConnection.HTTP_OK) { 209 String errorHeader = response.getHeaderField("Error"); 210 String errorBody = fetchResponseText(response); 211 throw new OsmApiException(response.getResponseCode(), errorHeader, errorBody, url.toString(), null, 212 response.getContentType()); 213 } 214 215 response.uncompressAccordingToContentDisposition(uncompressAccordingToContentDisposition); 216 return response.getContent(); 217 } catch (OsmTransferException e) { 218 throw e; 219 } catch (IOException e) { 220 throw new OsmTransferException(e); 221 } 222 } finally { 223 progressMonitor.invalidate(); 224 } 225 } 226 227 private static String fetchResponseText(final HttpClient.Response response) { 228 try { 229 return response.fetchContent(); 230 } catch (IOException e) { 231 Logging.error(e); 232 return tr("Reading error text failed."); 233 } 234 } 235 236 /** 237 * Allows subclasses to modify the request. 238 * @param request the prepared request 239 * @since 9308 240 */ 241 protected void adaptRequest(HttpClient request) { 242 } 243 244 /** 245 * Download OSM files from somewhere 246 * @param progressMonitor The progress monitor 247 * @return The corresponding dataset 248 * @throws OsmTransferException if any error occurs 249 */ 250 public abstract DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException; 251 252 /** 253 * Download compressed OSM files from somewhere 254 * @param progressMonitor The progress monitor 255 * @param compression compression to use 256 * @return The corresponding dataset 257 * @throws OsmTransferException if any error occurs 258 * @since 13352 259 */ 260 public DataSet parseOsm(ProgressMonitor progressMonitor, Compression compression) throws OsmTransferException { 261 return null; 262 } 263 264 /** 265 * Download OSM Change uncompressed files from somewhere 266 * @param progressMonitor The progress monitor 267 * @return The corresponding dataset 268 * @throws OsmTransferException if any error occurs 269 */ 270 public DataSet parseOsmChange(ProgressMonitor progressMonitor) throws OsmTransferException { 271 return null; 272 } 273 274 /** 275 * Download OSM Change compressed files from somewhere 276 * @param progressMonitor The progress monitor 277 * @param compression compression to use 278 * @return The corresponding dataset 279 * @throws OsmTransferException if any error occurs 280 * @since 13352 281 */ 282 public DataSet parseOsmChange(ProgressMonitor progressMonitor, Compression compression) throws OsmTransferException { 283 return null; 284 } 285 286 /** 287 * Retrieve raw gps waypoints from the server API. 288 * @param progressMonitor The progress monitor 289 * @return The corresponding GPX tracks 290 * @throws OsmTransferException if any error occurs 291 */ 292 public GpxData parseRawGps(ProgressMonitor progressMonitor) throws OsmTransferException { 293 return null; 294 } 295 296 /** 297 * Retrieve compressed GPX files from somewhere. 298 * @param progressMonitor The progress monitor 299 * @param compression compression to use 300 * @return The corresponding GPX tracks 301 * @throws OsmTransferException if any error occurs 302 * @since 13352 303 */ 304 public GpxData parseRawGps(ProgressMonitor progressMonitor, Compression compression) throws OsmTransferException { 305 return null; 306 } 307 308 /** 309 * Returns true if this reader is adding authentication credentials to the read 310 * request sent to the server. 311 * 312 * @return true if this reader is adding authentication credentials to the read 313 * request sent to the server 314 */ 315 public boolean isDoAuthenticate() { 316 return doAuthenticate; 317 } 318 319 /** 320 * Sets whether this reader adds authentication credentials to the read 321 * request sent to the server. 322 * 323 * @param doAuthenticate true if this reader adds authentication credentials to the read 324 * request sent to the server 325 */ 326 public void setDoAuthenticate(boolean doAuthenticate) { 327 this.doAuthenticate = doAuthenticate; 328 } 329 330 /** 331 * Determines if the GPX data has been parsed properly. 332 * @return true if the GPX data has been parsed properly, false otherwise 333 * @see GpxReader#parse 334 */ 335 public final boolean isGpxParsedProperly() { 336 return gpxParsedProperly; 337 } 338 339 /** 340 * Downloads notes from the API, given API limit parameters 341 * 342 * @param noteLimit How many notes to download. 343 * @param daysClosed Return notes closed this many days in the past. -1 means all notes, ever. 0 means only unresolved notes. 344 * @param progressMonitor Progress monitor for user feedback 345 * @return List of notes returned by the API 346 * @throws OsmTransferException if any errors happen 347 */ 348 public List<Note> parseNotes(int noteLimit, int daysClosed, ProgressMonitor progressMonitor) throws OsmTransferException { 349 return null; 350 } 351 352 /** 353 * Downloads notes from a given raw URL. The URL is assumed to be complete and no API limits are added 354 * 355 * @param progressMonitor progress monitor 356 * @return A list of notes parsed from the URL 357 * @throws OsmTransferException if any error occurs during dialog with OSM API 358 */ 359 public List<Note> parseRawNotes(final ProgressMonitor progressMonitor) throws OsmTransferException { 360 return null; 361 } 362 363 /** 364 * Download notes from a URL that contains a compressed notes dump file 365 * @param progressMonitor progress monitor 366 * @param compression compression to use 367 * @return A list of notes parsed from the URL 368 * @throws OsmTransferException if any error occurs during dialog with OSM API 369 * @since 13352 370 */ 371 public List<Note> parseRawNotes(ProgressMonitor progressMonitor, Compression compression) throws OsmTransferException { 372 return null; 373 } 374 375 /** 376 * Returns an attribute from the given DOM node. 377 * @param node DOM node 378 * @param name attribute name 379 * @return attribute value for the given attribute 380 * @since 12510 381 */ 382 protected static String getAttribute(Node node, String name) { 383 return node.getAttributes().getNamedItem(name).getNodeValue(); 384 } 385 386 /** 387 * DOM document parser. 388 * @param <R> resulting type 389 * @since 12510 390 */ 391 @FunctionalInterface 392 protected interface DomParser<R> { 393 /** 394 * Parses a given DOM document. 395 * @param doc DOM document 396 * @return parsed data 397 * @throws XmlParsingException if an XML parsing error occurs 398 */ 399 R parse(Document doc) throws XmlParsingException; 400 } 401 402 /** 403 * Fetches generic data from the DOM document resulting an API call. 404 * @param api the OSM API call 405 * @param subtask the subtask translated message 406 * @param parser the parser converting the DOM document (OSM API result) 407 * @param <T> data type 408 * @param monitor The progress monitor 409 * @param reason The reason to show on console. Can be {@code null} if no reason is given 410 * @return The converted data 411 * @throws OsmTransferException if something goes wrong 412 * @since 12510 413 */ 414 public <T> T fetchData(String api, String subtask, DomParser<T> parser, ProgressMonitor monitor, String reason) 415 throws OsmTransferException { 416 try { 417 monitor.beginTask(""); 418 monitor.indeterminateSubTask(subtask); 419 try (InputStream in = getInputStream(api, monitor.createSubTaskMonitor(1, true), reason)) { 420 return parser.parse(XmlUtils.parseSafeDOM(in)); 421 } 422 } catch (OsmTransferException e) { 423 throw e; 424 } catch (IOException | ParserConfigurationException | SAXException e) { 425 throw new OsmTransferException(e); 426 } finally { 427 monitor.finishTask(); 428 } 429 } 430}