001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.io;
003
004import static org.openstreetmap.josm.tools.I18n.tr;
005
006import java.io.IOException;
007import java.io.InputStream;
008import java.net.Authenticator.RequestorType;
009import java.net.HttpURLConnection;
010import java.net.MalformedURLException;
011import java.net.URL;
012import java.util.List;
013
014import javax.xml.parsers.ParserConfigurationException;
015
016import org.openstreetmap.josm.Main;
017import org.openstreetmap.josm.data.gpx.GpxData;
018import org.openstreetmap.josm.data.notes.Note;
019import org.openstreetmap.josm.data.osm.DataSet;
020import org.openstreetmap.josm.gui.progress.ProgressMonitor;
021import org.openstreetmap.josm.io.auth.CredentialsAgentException;
022import org.openstreetmap.josm.io.auth.CredentialsManager;
023import org.openstreetmap.josm.tools.HttpClient;
024import org.openstreetmap.josm.tools.Logging;
025import org.openstreetmap.josm.tools.Utils;
026import org.openstreetmap.josm.tools.XmlParsingException;
027import org.openstreetmap.josm.tools.XmlUtils;
028import org.w3c.dom.Document;
029import org.w3c.dom.Node;
030import org.xml.sax.SAXException;
031
032/**
033 * This DataReader reads directly from the REST API of the osm server.
034 *
035 * It supports plain text transfer as well as gzip or deflate encoded transfers;
036 * if compressed transfers are unwanted, set property osm-server.use-compression
037 * to false.
038 *
039 * @author imi
040 */
041public abstract class OsmServerReader extends OsmConnection {
042    private final OsmApi api = OsmApi.getOsmApi();
043    private boolean doAuthenticate;
044    protected boolean gpxParsedProperly;
045
046    /**
047     * Constructs a new {@code OsmServerReader}.
048     */
049    public OsmServerReader() {
050        try {
051            doAuthenticate = OsmApi.isUsingOAuth() && CredentialsManager.getInstance().lookupOAuthAccessToken() != null;
052        } catch (CredentialsAgentException e) {
053            Logging.warn(e);
054        }
055    }
056
057    /**
058     * Open a connection to the given url and return a reader on the input stream
059     * from that connection. In case of user cancel, return <code>null</code>.
060     * Relative URL's are directed to API base URL.
061     * @param urlStr The url to connect to.
062     * @param progressMonitor progress monitoring and abort handler
063     * @return A reader reading the input stream (servers answer) or <code>null</code>.
064     * @throws OsmTransferException if data transfer errors occur
065     */
066    protected InputStream getInputStream(String urlStr, ProgressMonitor progressMonitor) throws OsmTransferException {
067        return getInputStream(urlStr, progressMonitor, null);
068    }
069
070    /**
071     * Open a connection to the given url and return a reader on the input stream
072     * from that connection. In case of user cancel, return <code>null</code>.
073     * Relative URL's are directed to API base URL.
074     * @param urlStr The url to connect to.
075     * @param progressMonitor progress monitoring and abort handler
076     * @param reason The reason to show on console. Can be {@code null} if no reason is given
077     * @return A reader reading the input stream (servers answer) or <code>null</code>.
078     * @throws OsmTransferException if data transfer errors occur
079     */
080    protected InputStream getInputStream(String urlStr, ProgressMonitor progressMonitor, String reason) throws OsmTransferException {
081        try {
082            api.initialize(progressMonitor);
083            String url = urlStr.startsWith("http") ? urlStr : (getBaseUrl() + urlStr);
084            return getInputStreamRaw(url, progressMonitor, reason);
085        } finally {
086            progressMonitor.invalidate();
087        }
088    }
089
090    /**
091     * Return the base URL for relative URL requests
092     * @return base url of API
093     */
094    protected String getBaseUrl() {
095        return api.getBaseUrl();
096    }
097
098    /**
099     * Open a connection to the given url and return a reader on the input stream
100     * from that connection. In case of user cancel, return <code>null</code>.
101     * @param urlStr The exact url to connect to.
102     * @param progressMonitor progress monitoring and abort handler
103     * @return An reader reading the input stream (servers answer) or <code>null</code>.
104     * @throws OsmTransferException if data transfer errors occur
105     */
106    protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor) throws OsmTransferException {
107        return getInputStreamRaw(urlStr, progressMonitor, null);
108    }
109
110    /**
111     * Open a connection to the given url and return a reader on the input stream
112     * from that connection. In case of user cancel, return <code>null</code>.
113     * @param urlStr The exact url to connect to.
114     * @param progressMonitor progress monitoring and abort handler
115     * @param reason The reason to show on console. Can be {@code null} if no reason is given
116     * @return An reader reading the input stream (servers answer) or <code>null</code>.
117     * @throws OsmTransferException if data transfer errors occur
118     */
119    protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason) throws OsmTransferException {
120        return getInputStreamRaw(urlStr, progressMonitor, reason, false);
121    }
122
123    /**
124     * Open a connection to the given url (if HTTP, trough a GET request) and return a reader on the input stream
125     * from that connection. In case of user cancel, return <code>null</code>.
126     * @param urlStr The exact url to connect to.
127     * @param progressMonitor progress monitoring and abort handler
128     * @param reason The reason to show on console. Can be {@code null} if no reason is given
129     * @param uncompressAccordingToContentDisposition Whether to inspect the HTTP header {@code Content-Disposition}
130     *                                                for {@code filename} and uncompress a gzip/bzip2/xz/zip stream.
131     * @return An reader reading the input stream (servers answer) or <code>null</code>.
132     * @throws OsmTransferException if data transfer errors occur
133     */
134    protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason,
135            boolean uncompressAccordingToContentDisposition) throws OsmTransferException {
136        return getInputStreamRaw(urlStr, progressMonitor, reason, uncompressAccordingToContentDisposition, "GET", null);
137    }
138
139    /**
140     * Open a connection to the given url (if HTTP, with the specified method) and return a reader on the input stream
141     * from that connection. In case of user cancel, return <code>null</code>.
142     * @param urlStr The exact url to connect to.
143     * @param progressMonitor progress monitoring and abort handler
144     * @param reason The reason to show on console. Can be {@code null} if no reason is given
145     * @param uncompressAccordingToContentDisposition Whether to inspect the HTTP header {@code Content-Disposition}
146     *                                                for {@code filename} and uncompress a gzip/bzip2/xz/zip stream.
147     * @param httpMethod HTTP method ("GET", "POST" or "PUT")
148     * @param requestBody HTTP request body (for "POST" and "PUT" methods only). Must be null for "GET" method.
149     * @return An reader reading the input stream (servers answer) or <code>null</code>.
150     * @throws OsmTransferException if data transfer errors occur
151     * @since 12596
152     */
153    @SuppressWarnings("resource")
154    protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason,
155            boolean uncompressAccordingToContentDisposition, String httpMethod, byte[] requestBody) throws OsmTransferException {
156        try {
157            OnlineResource.JOSM_WEBSITE.checkOfflineAccess(urlStr, Main.getJOSMWebsite());
158            OnlineResource.OSM_API.checkOfflineAccess(urlStr, OsmApi.getOsmApi().getServerUrl());
159
160            URL url = null;
161            try {
162                url = new URL(urlStr.replace(" ", "%20"));
163            } catch (MalformedURLException e) {
164                throw new OsmTransferException(e);
165            }
166
167            String protocol = url.getProtocol();
168            if ("file".equals(protocol) || "jar".equals(protocol)) {
169                try {
170                    return Utils.openStream(url);
171                } catch (IOException e) {
172                    throw new OsmTransferException(e);
173                }
174            }
175
176            final HttpClient client = HttpClient.create(url, httpMethod)
177                    .setFinishOnCloseOutput(false)
178                    .setReasonForRequest(reason)
179                    .setOutputMessage(tr("Downloading data..."))
180                    .setRequestBody(requestBody);
181            activeConnection = client;
182            adaptRequest(client);
183            if (doAuthenticate) {
184                addAuth(client);
185            }
186            if (cancel)
187                throw new OsmTransferCanceledException("Operation canceled");
188
189            final HttpClient.Response response;
190            try {
191                response = client.connect(progressMonitor);
192            } catch (IOException e) {
193                Logging.error(e);
194                OsmTransferException ote = new OsmTransferException(
195                        tr("Could not connect to the OSM server. Please check your internet connection."), e);
196                ote.setUrl(url.toString());
197                throw ote;
198            }
199            try {
200                if (response.getResponseCode() == HttpURLConnection.HTTP_UNAUTHORIZED) {
201                    CredentialsManager.getInstance().purgeCredentialsCache(RequestorType.SERVER);
202                    throw new OsmApiException(HttpURLConnection.HTTP_UNAUTHORIZED, null, null);
203                }
204
205                if (response.getResponseCode() == HttpURLConnection.HTTP_PROXY_AUTH)
206                    throw new OsmTransferCanceledException("Proxy Authentication Required");
207
208                if (response.getResponseCode() != HttpURLConnection.HTTP_OK) {
209                    String errorHeader = response.getHeaderField("Error");
210                    String errorBody = fetchResponseText(response);
211                    throw new OsmApiException(response.getResponseCode(), errorHeader, errorBody, url.toString(), null,
212                            response.getContentType());
213                }
214
215                response.uncompressAccordingToContentDisposition(uncompressAccordingToContentDisposition);
216                return response.getContent();
217            } catch (OsmTransferException e) {
218                throw e;
219            } catch (IOException e) {
220                throw new OsmTransferException(e);
221            }
222        } finally {
223            progressMonitor.invalidate();
224        }
225    }
226
227    private static String fetchResponseText(final HttpClient.Response response) {
228        try {
229            return response.fetchContent();
230        } catch (IOException e) {
231            Logging.error(e);
232            return tr("Reading error text failed.");
233        }
234    }
235
236    /**
237     * Allows subclasses to modify the request.
238     * @param request the prepared request
239     * @since 9308
240     */
241    protected void adaptRequest(HttpClient request) {
242    }
243
244    /**
245     * Download OSM files from somewhere
246     * @param progressMonitor The progress monitor
247     * @return The corresponding dataset
248     * @throws OsmTransferException if any error occurs
249     */
250    public abstract DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException;
251
252    /**
253     * Download compressed OSM files from somewhere
254     * @param progressMonitor The progress monitor
255     * @param compression compression to use
256     * @return The corresponding dataset
257     * @throws OsmTransferException if any error occurs
258     * @since 13352
259     */
260    public DataSet parseOsm(ProgressMonitor progressMonitor, Compression compression) throws OsmTransferException {
261        return null;
262    }
263
264    /**
265     * Download OSM Change uncompressed files from somewhere
266     * @param progressMonitor The progress monitor
267     * @return The corresponding dataset
268     * @throws OsmTransferException if any error occurs
269     */
270    public DataSet parseOsmChange(ProgressMonitor progressMonitor) throws OsmTransferException {
271        return null;
272    }
273
274    /**
275     * Download OSM Change compressed files from somewhere
276     * @param progressMonitor The progress monitor
277     * @param compression compression to use
278     * @return The corresponding dataset
279     * @throws OsmTransferException if any error occurs
280     * @since 13352
281     */
282    public DataSet parseOsmChange(ProgressMonitor progressMonitor, Compression compression) throws OsmTransferException {
283        return null;
284    }
285
286    /**
287     * Retrieve raw gps waypoints from the server API.
288     * @param progressMonitor The progress monitor
289     * @return The corresponding GPX tracks
290     * @throws OsmTransferException if any error occurs
291     */
292    public GpxData parseRawGps(ProgressMonitor progressMonitor) throws OsmTransferException {
293        return null;
294    }
295
296    /**
297     * Retrieve compressed GPX files from somewhere.
298     * @param progressMonitor The progress monitor
299     * @param compression compression to use
300     * @return The corresponding GPX tracks
301     * @throws OsmTransferException if any error occurs
302     * @since 13352
303     */
304    public GpxData parseRawGps(ProgressMonitor progressMonitor, Compression compression) throws OsmTransferException {
305        return null;
306    }
307
308    /**
309     * Returns true if this reader is adding authentication credentials to the read
310     * request sent to the server.
311     *
312     * @return true if this reader is adding authentication credentials to the read
313     * request sent to the server
314     */
315    public boolean isDoAuthenticate() {
316        return doAuthenticate;
317    }
318
319    /**
320     * Sets whether this reader adds authentication credentials to the read
321     * request sent to the server.
322     *
323     * @param doAuthenticate  true if  this reader adds authentication credentials to the read
324     * request sent to the server
325     */
326    public void setDoAuthenticate(boolean doAuthenticate) {
327        this.doAuthenticate = doAuthenticate;
328    }
329
330    /**
331     * Determines if the GPX data has been parsed properly.
332     * @return true if the GPX data has been parsed properly, false otherwise
333     * @see GpxReader#parse
334     */
335    public final boolean isGpxParsedProperly() {
336        return gpxParsedProperly;
337    }
338
339    /**
340     * Downloads notes from the API, given API limit parameters
341     *
342     * @param noteLimit How many notes to download.
343     * @param daysClosed Return notes closed this many days in the past. -1 means all notes, ever. 0 means only unresolved notes.
344     * @param progressMonitor Progress monitor for user feedback
345     * @return List of notes returned by the API
346     * @throws OsmTransferException if any errors happen
347     */
348    public List<Note> parseNotes(int noteLimit, int daysClosed, ProgressMonitor progressMonitor) throws OsmTransferException {
349        return null;
350    }
351
352    /**
353     * Downloads notes from a given raw URL. The URL is assumed to be complete and no API limits are added
354     *
355     * @param progressMonitor progress monitor
356     * @return A list of notes parsed from the URL
357     * @throws OsmTransferException if any error occurs during dialog with OSM API
358     */
359    public List<Note> parseRawNotes(final ProgressMonitor progressMonitor) throws OsmTransferException {
360        return null;
361    }
362
363    /**
364     * Download notes from a URL that contains a compressed notes dump file
365     * @param progressMonitor progress monitor
366     * @param compression compression to use
367     * @return A list of notes parsed from the URL
368     * @throws OsmTransferException if any error occurs during dialog with OSM API
369     * @since 13352
370     */
371    public List<Note> parseRawNotes(ProgressMonitor progressMonitor, Compression compression) throws OsmTransferException {
372        return null;
373    }
374
375    /**
376     * Returns an attribute from the given DOM node.
377     * @param node DOM node
378     * @param name attribute name
379     * @return attribute value for the given attribute
380     * @since 12510
381     */
382    protected static String getAttribute(Node node, String name) {
383        return node.getAttributes().getNamedItem(name).getNodeValue();
384    }
385
386    /**
387     * DOM document parser.
388     * @param <R> resulting type
389     * @since 12510
390     */
391    @FunctionalInterface
392    protected interface DomParser<R> {
393        /**
394         * Parses a given DOM document.
395         * @param doc DOM document
396         * @return parsed data
397         * @throws XmlParsingException if an XML parsing error occurs
398         */
399        R parse(Document doc) throws XmlParsingException;
400    }
401
402    /**
403     * Fetches generic data from the DOM document resulting an API call.
404     * @param api the OSM API call
405     * @param subtask the subtask translated message
406     * @param parser the parser converting the DOM document (OSM API result)
407     * @param <T> data type
408     * @param monitor The progress monitor
409     * @param reason The reason to show on console. Can be {@code null} if no reason is given
410     * @return The converted data
411     * @throws OsmTransferException if something goes wrong
412     * @since 12510
413     */
414    public <T> T fetchData(String api, String subtask, DomParser<T> parser, ProgressMonitor monitor, String reason)
415            throws OsmTransferException {
416        try {
417            monitor.beginTask("");
418            monitor.indeterminateSubTask(subtask);
419            try (InputStream in = getInputStream(api, monitor.createSubTaskMonitor(1, true), reason)) {
420                return parser.parse(XmlUtils.parseSafeDOM(in));
421            }
422        } catch (OsmTransferException e) {
423            throw e;
424        } catch (IOException | ParserConfigurationException | SAXException e) {
425            throw new OsmTransferException(e);
426        } finally {
427            monitor.finishTask();
428        }
429    }
430}