001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.io;
003
004import static org.openstreetmap.josm.tools.I18n.tr;
005
006import java.io.IOException;
007import java.io.InputStream;
008import java.nio.charset.StandardCharsets;
009import java.time.Duration;
010import java.time.LocalDateTime;
011import java.time.Period;
012import java.time.ZoneOffset;
013import java.util.Arrays;
014import java.util.EnumMap;
015import java.util.List;
016import java.util.Locale;
017import java.util.Map;
018import java.util.NoSuchElementException;
019import java.util.Objects;
020import java.util.concurrent.ConcurrentHashMap;
021import java.util.concurrent.TimeUnit;
022import java.util.regex.Matcher;
023import java.util.regex.Pattern;
024
025import javax.xml.stream.XMLStreamConstants;
026import javax.xml.stream.XMLStreamException;
027
028import org.openstreetmap.josm.data.Bounds;
029import org.openstreetmap.josm.data.DataSource;
030import org.openstreetmap.josm.data.coor.LatLon;
031import org.openstreetmap.josm.data.osm.BBox;
032import org.openstreetmap.josm.data.osm.DataSet;
033import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
034import org.openstreetmap.josm.data.osm.PrimitiveId;
035import org.openstreetmap.josm.data.preferences.BooleanProperty;
036import org.openstreetmap.josm.data.preferences.ListProperty;
037import org.openstreetmap.josm.data.preferences.StringProperty;
038import org.openstreetmap.josm.gui.progress.ProgressMonitor;
039import org.openstreetmap.josm.io.NameFinder.SearchResult;
040import org.openstreetmap.josm.tools.HttpClient;
041import org.openstreetmap.josm.tools.Logging;
042import org.openstreetmap.josm.tools.UncheckedParseException;
043import org.openstreetmap.josm.tools.Utils;
044
045/**
046 * Read content from an Overpass server.
047 *
048 * @since 8744
049 */
050public class OverpassDownloadReader extends BoundingBoxDownloader {
051
052    /**
053     * Property for current Overpass server.
054     * @since 12816
055     */
056    public static final StringProperty OVERPASS_SERVER = new StringProperty("download.overpass.server",
057            "https://overpass-api.de/api/");
058    /**
059     * Property for list of known Overpass servers.
060     * @since 12816
061     */
062    public static final ListProperty OVERPASS_SERVER_HISTORY = new ListProperty("download.overpass.servers",
063            Arrays.asList("https://overpass-api.de/api/", "http://overpass.openstreetmap.ru/cgi/"));
064    /**
065     * Property to determine if Overpass API should be used for multi-fetch download.
066     * @since 12816
067     */
068    public static final BooleanProperty FOR_MULTI_FETCH = new BooleanProperty("download.overpass.for-multi-fetch", false);
069
070    private static final String DATA_PREFIX = "?data=";
071
072    static final class OverpassOsmReader extends OsmReader {
073        @Override
074        protected void parseUnknown(boolean printWarning) throws XMLStreamException {
075            if ("remark".equals(parser.getLocalName()) && parser.getEventType() == XMLStreamConstants.START_ELEMENT) {
076                final String text = parser.getElementText();
077                if (text.contains("runtime error")) {
078                    throw new XMLStreamException(text);
079                }
080            }
081            super.parseUnknown(printWarning);
082        }
083    }
084
085    /**
086     * Possible Overpass API output format, with the {@code [out:<directive>]} statement.
087     * @since 11916
088     */
089    public enum OverpassOutpoutFormat {
090        /** Default output format: plain OSM XML */
091        OSM_XML("xml"),
092        /** OSM JSON format (not GeoJson) */
093        OSM_JSON("json"),
094        /** CSV, see https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#Output_Format_.28out.29 */
095        CSV("csv"),
096        /** Custom, see https://overpass-api.de/output_formats.html#custom */
097        CUSTOM("custom"),
098        /** Popup, see https://overpass-api.de/output_formats.html#popup */
099        POPUP("popup"),
100        /** PBF, see https://josm.openstreetmap.de/ticket/14653 */
101        PBF("pbf");
102
103        private final String directive;
104
105        OverpassOutpoutFormat(String directive) {
106            this.directive = directive;
107        }
108
109        /**
110         * Returns the directive used in {@code [out:<directive>]} statement.
111         * @return the directive used in {@code [out:<directive>]} statement
112         */
113        public String getDirective() {
114            return directive;
115        }
116
117        /**
118         * Returns the {@code OverpassOutpoutFormat} matching the given directive.
119         * @param directive directive used in {@code [out:<directive>]} statement
120         * @return {@code OverpassOutpoutFormat} matching the given directive
121         * @throws IllegalArgumentException in case of invalid directive
122         */
123        static OverpassOutpoutFormat from(String directive) {
124            for (OverpassOutpoutFormat oof : values()) {
125                if (oof.directive.equals(directive)) {
126                    return oof;
127                }
128            }
129            throw new IllegalArgumentException(directive);
130        }
131    }
132
133    static final Pattern OUTPUT_FORMAT_STATEMENT = Pattern.compile(".*\\[out:([a-z]{3,})\\].*", Pattern.DOTALL);
134
135    static final Map<OverpassOutpoutFormat, Class<? extends AbstractReader>> outputFormatReaders = new ConcurrentHashMap<>();
136
137    final String overpassServer;
138    final String overpassQuery;
139
140    /**
141     * Constructs a new {@code OverpassDownloadReader}.
142     *
143     * @param downloadArea   The area to download
144     * @param overpassServer The Overpass server to use
145     * @param overpassQuery  The Overpass query
146     */
147    public OverpassDownloadReader(Bounds downloadArea, String overpassServer, String overpassQuery) {
148        super(downloadArea);
149        setDoAuthenticate(false);
150        this.overpassServer = overpassServer;
151        this.overpassQuery = overpassQuery.trim();
152    }
153
154    /**
155     * Registers an OSM reader for the given Overpass output format.
156     * @param format Overpass output format
157     * @param readerClass OSM reader class
158     * @return the previous value associated with {@code format}, or {@code null} if there was no mapping
159     */
160    public static final Class<? extends AbstractReader> registerOverpassOutpoutFormatReader(
161            OverpassOutpoutFormat format, Class<? extends AbstractReader> readerClass) {
162        return outputFormatReaders.put(Objects.requireNonNull(format), Objects.requireNonNull(readerClass));
163    }
164
165    static {
166        registerOverpassOutpoutFormatReader(OverpassOutpoutFormat.OSM_XML, OverpassOsmReader.class);
167    }
168
169    @Override
170    protected String getBaseUrl() {
171        return overpassServer;
172    }
173
174    @Override
175    protected String getRequestForBbox(double lon1, double lat1, double lon2, double lat2) {
176        if (overpassQuery.isEmpty())
177            return super.getRequestForBbox(lon1, lat1, lon2, lat2);
178        else {
179            final String query = this.overpassQuery
180                    .replace("{{bbox}}", bbox(lon1, lat1, lon2, lat2))
181                    .replace("{{center}}", center(lon1, lat1, lon2, lat2));
182            final String expandedOverpassQuery = expandExtendedQueries(query);
183            return "interpreter" + DATA_PREFIX + Utils.encodeUrl(expandedOverpassQuery);
184        }
185    }
186
187    /**
188     * Evaluates some features of overpass turbo extended query syntax.
189     * See https://wiki.openstreetmap.org/wiki/Overpass_turbo/Extended_Overpass_Turbo_Queries
190     * @param query unexpanded query
191     * @return expanded query
192     */
193    static String expandExtendedQueries(String query) {
194        final StringBuffer sb = new StringBuffer();
195        final Matcher matcher = Pattern.compile("\\{\\{(date|geocodeArea|geocodeBbox|geocodeCoords|geocodeId):([^}]+)\\}\\}").matcher(query);
196        while (matcher.find()) {
197            try {
198                switch (matcher.group(1)) {
199                    case "date":
200                        matcher.appendReplacement(sb, date(matcher.group(2), LocalDateTime.now()));
201                        break;
202                    case "geocodeArea":
203                        matcher.appendReplacement(sb, geocodeArea(matcher.group(2)));
204                        break;
205                    case "geocodeBbox":
206                        matcher.appendReplacement(sb, geocodeBbox(matcher.group(2)));
207                        break;
208                    case "geocodeCoords":
209                        matcher.appendReplacement(sb, geocodeCoords(matcher.group(2)));
210                        break;
211                    case "geocodeId":
212                        matcher.appendReplacement(sb, geocodeId(matcher.group(2)));
213                        break;
214                    default:
215                        Logging.warn("Unsupported syntax: " + matcher.group(1));
216                }
217            } catch (UncheckedParseException | IOException | NoSuchElementException | IndexOutOfBoundsException ex) {
218                final String msg = tr("Failed to evaluate {0}", matcher.group());
219                Logging.log(Logging.LEVEL_WARN, msg, ex);
220                matcher.appendReplacement(sb, "// " + msg + "\n");
221            }
222        }
223        matcher.appendTail(sb);
224        return sb.toString();
225    }
226
227    static String bbox(double lon1, double lat1, double lon2, double lat2) {
228        return lat1 + "," + lon1 + "," + lat2 + "," + lon2;
229    }
230
231    static String center(double lon1, double lat1, double lon2, double lat2) {
232        LatLon c = new BBox(lon1, lat1, lon2, lat2).getCenter();
233        return c.lat()+ "," + c.lon();
234    }
235
236    static String date(String humanDuration, LocalDateTime from) {
237        // Convert to ISO 8601. Replace months by X temporarily to avoid conflict with minutes
238        String duration = humanDuration.toLowerCase(Locale.ENGLISH).replace(" ", "")
239                .replaceAll("years?", "Y").replaceAll("months?", "X").replaceAll("weeks?", "W")
240                .replaceAll("days?", "D").replaceAll("hours?", "H").replaceAll("minutes?", "M").replaceAll("seconds?", "S");
241        Matcher matcher = Pattern.compile(
242                "((?:[0-9]+Y)?(?:[0-9]+X)?(?:[0-9]+W)?)"+
243                "((?:[0-9]+D)?)" +
244                "((?:[0-9]+H)?(?:[0-9]+M)?(?:[0-9]+(?:[.,][0-9]{0,9})?S)?)?").matcher(duration);
245        boolean javaPer = false;
246        boolean javaDur = false;
247        if (matcher.matches()) {
248            javaPer = matcher.group(1) != null && !matcher.group(1).isEmpty();
249            javaDur = matcher.group(3) != null && !matcher.group(3).isEmpty();
250            duration = 'P' + matcher.group(1).replace('X', 'M') + matcher.group(2);
251            if (javaDur) {
252                duration += 'T' + matcher.group(3);
253            }
254        }
255
256        // Duration is now a full ISO 8601 duration string. Unfortunately Java does not allow to parse it entirely.
257        // We must split the "period" (years, months, weeks, days) from the "duration" (days, hours, minutes, seconds).
258        Period p = null;
259        Duration d = null;
260        int idx = duration.indexOf('T');
261        if (javaPer) {
262            p = Period.parse(javaDur ? duration.substring(0, idx) : duration);
263        }
264        if (javaDur) {
265            d = Duration.parse(javaPer ? 'P' + duration.substring(idx, duration.length()) : duration);
266        } else if (!javaPer) {
267            d = Duration.parse(duration);
268        }
269
270        // Now that period and duration are known, compute the correct date/time
271        LocalDateTime dt = from;
272        if (p != null) {
273            dt = dt.minus(p);
274        }
275        if (d != null) {
276            dt = dt.minus(d);
277        }
278
279        // Returns the date/time formatted in ISO 8601
280        return dt.toInstant(ZoneOffset.UTC).toString();
281    }
282
283    private static SearchResult searchName(String area) throws IOException {
284        return searchName(NameFinder.queryNominatim(area));
285    }
286
287    static SearchResult searchName(List<SearchResult> results) {
288        return results.stream().filter(
289                x -> !OsmPrimitiveType.NODE.equals(x.getOsmId().getType())).iterator().next();
290    }
291
292    static String geocodeArea(String area) throws IOException {
293        // Offsets defined in https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#By_element_id
294        final EnumMap<OsmPrimitiveType, Long> idOffset = new EnumMap<>(OsmPrimitiveType.class);
295        idOffset.put(OsmPrimitiveType.NODE, 0L);
296        idOffset.put(OsmPrimitiveType.WAY, 2_400_000_000L);
297        idOffset.put(OsmPrimitiveType.RELATION, 3_600_000_000L);
298        final PrimitiveId osmId = searchName(area).getOsmId();
299        Logging.debug("Area '{0}' resolved to {1}", area, osmId);
300        return String.format("area(%d)", osmId.getUniqueId() + idOffset.get(osmId.getType()));
301    }
302
303    static String geocodeBbox(String area) throws IOException {
304        Bounds bounds = searchName(area).getBounds();
305        return bounds.getMinLat() + "," + bounds.getMinLon() + "," + bounds.getMaxLat() + "," + bounds.getMaxLon();
306    }
307
308    static String geocodeCoords(String area) throws IOException {
309        SearchResult result = searchName(area);
310        return result.getLat() + "," + result.getLon();
311    }
312
313    static String geocodeId(String area) throws IOException {
314        PrimitiveId osmId = searchName(area).getOsmId();
315        return String.format("%s(%d)", osmId.getType().getAPIName(), osmId.getUniqueId());
316    }
317
318    @Override
319    protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason,
320                                            boolean uncompressAccordingToContentDisposition) throws OsmTransferException {
321        try {
322            int index = urlStr.indexOf(DATA_PREFIX);
323            // Make an HTTP POST request instead of a simple GET, allows more complex queries
324            return super.getInputStreamRaw(urlStr.substring(0, index),
325                    progressMonitor, reason, uncompressAccordingToContentDisposition,
326                    "POST", Utils.decodeUrl(urlStr.substring(index + DATA_PREFIX.length())).getBytes(StandardCharsets.UTF_8));
327        } catch (OsmApiException ex) {
328            final String errorIndicator = "Error</strong>: ";
329            if (ex.getMessage() != null && ex.getMessage().contains(errorIndicator)) {
330                final String errorPlusRest = ex.getMessage().split(errorIndicator)[1];
331                if (errorPlusRest != null) {
332                    ex.setErrorHeader(errorPlusRest.split("</")[0].replaceAll(".*::request_read_and_idx::", ""));
333                }
334            }
335            throw ex;
336        }
337    }
338
339    @Override
340    protected void adaptRequest(HttpClient request) {
341        // see https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#timeout
342        final Matcher timeoutMatcher = Pattern.compile("\\[timeout:(\\d+)\\]").matcher(overpassQuery);
343        final int timeout;
344        if (timeoutMatcher.find()) {
345            timeout = (int) TimeUnit.SECONDS.toMillis(Integer.parseInt(timeoutMatcher.group(1)));
346        } else {
347            timeout = (int) TimeUnit.MINUTES.toMillis(3);
348        }
349        request.setConnectTimeout(timeout);
350        request.setReadTimeout(timeout);
351    }
352
353    @Override
354    protected String getTaskName() {
355        return tr("Contacting Server...");
356    }
357
358    @Override
359    protected DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
360        AbstractReader reader = null;
361        Matcher m = OUTPUT_FORMAT_STATEMENT.matcher(overpassQuery);
362        if (m.matches()) {
363            Class<? extends AbstractReader> readerClass = outputFormatReaders.get(OverpassOutpoutFormat.from(m.group(1)));
364            if (readerClass != null) {
365                try {
366                    reader = readerClass.getDeclaredConstructor().newInstance();
367                } catch (ReflectiveOperationException | IllegalArgumentException | SecurityException e) {
368                    Logging.error(e);
369                }
370            }
371        }
372        if (reader == null) {
373            reader = new OverpassOsmReader();
374        }
375        return reader.doParseDataSet(source, progressMonitor);
376    }
377
378    @Override
379    public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException {
380
381        DataSet ds = super.parseOsm(progressMonitor);
382
383        // add bounds if necessary (note that Overpass API does not return bounds in the response XML)
384        if (ds != null && ds.getDataSources().isEmpty() && overpassQuery.contains("{{bbox}}")) {
385            if (crosses180th) {
386                Bounds bounds = new Bounds(lat1, lon1, lat2, 180.0);
387                DataSource src = new DataSource(bounds, getBaseUrl());
388                ds.addDataSource(src);
389
390                bounds = new Bounds(lat1, -180.0, lat2, lon2);
391                src = new DataSource(bounds, getBaseUrl());
392                ds.addDataSource(src);
393            } else {
394                Bounds bounds = new Bounds(lat1, lon1, lat2, lon2);
395                DataSource src = new DataSource(bounds, getBaseUrl());
396                ds.addDataSource(src);
397            }
398        }
399
400        return ds;
401    }
402
403    /**
404     * Fixes Overpass API query to make sure it will be accepted by JOSM.
405     * @param query Overpass query to check
406     * @return fixed query
407     * @since 13335
408     */
409    public static String fixQuery(String query) {
410        return query == null ? query : query
411                .replaceAll("out( body| skel| ids)?( id| qt)?;", "out meta$2;")
412                .replaceAll("(?s)\\[out:(json|csv)[^\\]]*\\]", "[out:xml]");
413    }
414}