001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.io; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.io.IOException; 007import java.io.InputStream; 008import java.nio.charset.StandardCharsets; 009import java.time.Duration; 010import java.time.LocalDateTime; 011import java.time.Period; 012import java.time.ZoneOffset; 013import java.util.Arrays; 014import java.util.EnumMap; 015import java.util.List; 016import java.util.Locale; 017import java.util.Map; 018import java.util.NoSuchElementException; 019import java.util.Objects; 020import java.util.concurrent.ConcurrentHashMap; 021import java.util.concurrent.TimeUnit; 022import java.util.regex.Matcher; 023import java.util.regex.Pattern; 024 025import javax.xml.stream.XMLStreamConstants; 026import javax.xml.stream.XMLStreamException; 027 028import org.openstreetmap.josm.data.Bounds; 029import org.openstreetmap.josm.data.DataSource; 030import org.openstreetmap.josm.data.coor.LatLon; 031import org.openstreetmap.josm.data.osm.BBox; 032import org.openstreetmap.josm.data.osm.DataSet; 033import org.openstreetmap.josm.data.osm.OsmPrimitiveType; 034import org.openstreetmap.josm.data.osm.PrimitiveId; 035import org.openstreetmap.josm.data.preferences.BooleanProperty; 036import org.openstreetmap.josm.data.preferences.ListProperty; 037import org.openstreetmap.josm.data.preferences.StringProperty; 038import org.openstreetmap.josm.gui.progress.ProgressMonitor; 039import org.openstreetmap.josm.io.NameFinder.SearchResult; 040import org.openstreetmap.josm.tools.HttpClient; 041import org.openstreetmap.josm.tools.Logging; 042import org.openstreetmap.josm.tools.UncheckedParseException; 043import org.openstreetmap.josm.tools.Utils; 044 045/** 046 * Read content from an Overpass server. 047 * 048 * @since 8744 049 */ 050public class OverpassDownloadReader extends BoundingBoxDownloader { 051 052 /** 053 * Property for current Overpass server. 054 * @since 12816 055 */ 056 public static final StringProperty OVERPASS_SERVER = new StringProperty("download.overpass.server", 057 "https://overpass-api.de/api/"); 058 /** 059 * Property for list of known Overpass servers. 060 * @since 12816 061 */ 062 public static final ListProperty OVERPASS_SERVER_HISTORY = new ListProperty("download.overpass.servers", 063 Arrays.asList("https://overpass-api.de/api/", "http://overpass.openstreetmap.ru/cgi/")); 064 /** 065 * Property to determine if Overpass API should be used for multi-fetch download. 066 * @since 12816 067 */ 068 public static final BooleanProperty FOR_MULTI_FETCH = new BooleanProperty("download.overpass.for-multi-fetch", false); 069 070 private static final String DATA_PREFIX = "?data="; 071 072 static final class OverpassOsmReader extends OsmReader { 073 @Override 074 protected void parseUnknown(boolean printWarning) throws XMLStreamException { 075 if ("remark".equals(parser.getLocalName()) && parser.getEventType() == XMLStreamConstants.START_ELEMENT) { 076 final String text = parser.getElementText(); 077 if (text.contains("runtime error")) { 078 throw new XMLStreamException(text); 079 } 080 } 081 super.parseUnknown(printWarning); 082 } 083 } 084 085 /** 086 * Possible Overpass API output format, with the {@code [out:<directive>]} statement. 087 * @since 11916 088 */ 089 public enum OverpassOutpoutFormat { 090 /** Default output format: plain OSM XML */ 091 OSM_XML("xml"), 092 /** OSM JSON format (not GeoJson) */ 093 OSM_JSON("json"), 094 /** CSV, see https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#Output_Format_.28out.29 */ 095 CSV("csv"), 096 /** Custom, see https://overpass-api.de/output_formats.html#custom */ 097 CUSTOM("custom"), 098 /** Popup, see https://overpass-api.de/output_formats.html#popup */ 099 POPUP("popup"), 100 /** PBF, see https://josm.openstreetmap.de/ticket/14653 */ 101 PBF("pbf"); 102 103 private final String directive; 104 105 OverpassOutpoutFormat(String directive) { 106 this.directive = directive; 107 } 108 109 /** 110 * Returns the directive used in {@code [out:<directive>]} statement. 111 * @return the directive used in {@code [out:<directive>]} statement 112 */ 113 public String getDirective() { 114 return directive; 115 } 116 117 /** 118 * Returns the {@code OverpassOutpoutFormat} matching the given directive. 119 * @param directive directive used in {@code [out:<directive>]} statement 120 * @return {@code OverpassOutpoutFormat} matching the given directive 121 * @throws IllegalArgumentException in case of invalid directive 122 */ 123 static OverpassOutpoutFormat from(String directive) { 124 for (OverpassOutpoutFormat oof : values()) { 125 if (oof.directive.equals(directive)) { 126 return oof; 127 } 128 } 129 throw new IllegalArgumentException(directive); 130 } 131 } 132 133 static final Pattern OUTPUT_FORMAT_STATEMENT = Pattern.compile(".*\\[out:([a-z]{3,})\\].*", Pattern.DOTALL); 134 135 static final Map<OverpassOutpoutFormat, Class<? extends AbstractReader>> outputFormatReaders = new ConcurrentHashMap<>(); 136 137 final String overpassServer; 138 final String overpassQuery; 139 140 /** 141 * Constructs a new {@code OverpassDownloadReader}. 142 * 143 * @param downloadArea The area to download 144 * @param overpassServer The Overpass server to use 145 * @param overpassQuery The Overpass query 146 */ 147 public OverpassDownloadReader(Bounds downloadArea, String overpassServer, String overpassQuery) { 148 super(downloadArea); 149 setDoAuthenticate(false); 150 this.overpassServer = overpassServer; 151 this.overpassQuery = overpassQuery.trim(); 152 } 153 154 /** 155 * Registers an OSM reader for the given Overpass output format. 156 * @param format Overpass output format 157 * @param readerClass OSM reader class 158 * @return the previous value associated with {@code format}, or {@code null} if there was no mapping 159 */ 160 public static final Class<? extends AbstractReader> registerOverpassOutpoutFormatReader( 161 OverpassOutpoutFormat format, Class<? extends AbstractReader> readerClass) { 162 return outputFormatReaders.put(Objects.requireNonNull(format), Objects.requireNonNull(readerClass)); 163 } 164 165 static { 166 registerOverpassOutpoutFormatReader(OverpassOutpoutFormat.OSM_XML, OverpassOsmReader.class); 167 } 168 169 @Override 170 protected String getBaseUrl() { 171 return overpassServer; 172 } 173 174 @Override 175 protected String getRequestForBbox(double lon1, double lat1, double lon2, double lat2) { 176 if (overpassQuery.isEmpty()) 177 return super.getRequestForBbox(lon1, lat1, lon2, lat2); 178 else { 179 final String query = this.overpassQuery 180 .replace("{{bbox}}", bbox(lon1, lat1, lon2, lat2)) 181 .replace("{{center}}", center(lon1, lat1, lon2, lat2)); 182 final String expandedOverpassQuery = expandExtendedQueries(query); 183 return "interpreter" + DATA_PREFIX + Utils.encodeUrl(expandedOverpassQuery); 184 } 185 } 186 187 /** 188 * Evaluates some features of overpass turbo extended query syntax. 189 * See https://wiki.openstreetmap.org/wiki/Overpass_turbo/Extended_Overpass_Turbo_Queries 190 * @param query unexpanded query 191 * @return expanded query 192 */ 193 static String expandExtendedQueries(String query) { 194 final StringBuffer sb = new StringBuffer(); 195 final Matcher matcher = Pattern.compile("\\{\\{(date|geocodeArea|geocodeBbox|geocodeCoords|geocodeId):([^}]+)\\}\\}").matcher(query); 196 while (matcher.find()) { 197 try { 198 switch (matcher.group(1)) { 199 case "date": 200 matcher.appendReplacement(sb, date(matcher.group(2), LocalDateTime.now())); 201 break; 202 case "geocodeArea": 203 matcher.appendReplacement(sb, geocodeArea(matcher.group(2))); 204 break; 205 case "geocodeBbox": 206 matcher.appendReplacement(sb, geocodeBbox(matcher.group(2))); 207 break; 208 case "geocodeCoords": 209 matcher.appendReplacement(sb, geocodeCoords(matcher.group(2))); 210 break; 211 case "geocodeId": 212 matcher.appendReplacement(sb, geocodeId(matcher.group(2))); 213 break; 214 default: 215 Logging.warn("Unsupported syntax: " + matcher.group(1)); 216 } 217 } catch (UncheckedParseException | IOException | NoSuchElementException | IndexOutOfBoundsException ex) { 218 final String msg = tr("Failed to evaluate {0}", matcher.group()); 219 Logging.log(Logging.LEVEL_WARN, msg, ex); 220 matcher.appendReplacement(sb, "// " + msg + "\n"); 221 } 222 } 223 matcher.appendTail(sb); 224 return sb.toString(); 225 } 226 227 static String bbox(double lon1, double lat1, double lon2, double lat2) { 228 return lat1 + "," + lon1 + "," + lat2 + "," + lon2; 229 } 230 231 static String center(double lon1, double lat1, double lon2, double lat2) { 232 LatLon c = new BBox(lon1, lat1, lon2, lat2).getCenter(); 233 return c.lat()+ "," + c.lon(); 234 } 235 236 static String date(String humanDuration, LocalDateTime from) { 237 // Convert to ISO 8601. Replace months by X temporarily to avoid conflict with minutes 238 String duration = humanDuration.toLowerCase(Locale.ENGLISH).replace(" ", "") 239 .replaceAll("years?", "Y").replaceAll("months?", "X").replaceAll("weeks?", "W") 240 .replaceAll("days?", "D").replaceAll("hours?", "H").replaceAll("minutes?", "M").replaceAll("seconds?", "S"); 241 Matcher matcher = Pattern.compile( 242 "((?:[0-9]+Y)?(?:[0-9]+X)?(?:[0-9]+W)?)"+ 243 "((?:[0-9]+D)?)" + 244 "((?:[0-9]+H)?(?:[0-9]+M)?(?:[0-9]+(?:[.,][0-9]{0,9})?S)?)?").matcher(duration); 245 boolean javaPer = false; 246 boolean javaDur = false; 247 if (matcher.matches()) { 248 javaPer = matcher.group(1) != null && !matcher.group(1).isEmpty(); 249 javaDur = matcher.group(3) != null && !matcher.group(3).isEmpty(); 250 duration = 'P' + matcher.group(1).replace('X', 'M') + matcher.group(2); 251 if (javaDur) { 252 duration += 'T' + matcher.group(3); 253 } 254 } 255 256 // Duration is now a full ISO 8601 duration string. Unfortunately Java does not allow to parse it entirely. 257 // We must split the "period" (years, months, weeks, days) from the "duration" (days, hours, minutes, seconds). 258 Period p = null; 259 Duration d = null; 260 int idx = duration.indexOf('T'); 261 if (javaPer) { 262 p = Period.parse(javaDur ? duration.substring(0, idx) : duration); 263 } 264 if (javaDur) { 265 d = Duration.parse(javaPer ? 'P' + duration.substring(idx, duration.length()) : duration); 266 } else if (!javaPer) { 267 d = Duration.parse(duration); 268 } 269 270 // Now that period and duration are known, compute the correct date/time 271 LocalDateTime dt = from; 272 if (p != null) { 273 dt = dt.minus(p); 274 } 275 if (d != null) { 276 dt = dt.minus(d); 277 } 278 279 // Returns the date/time formatted in ISO 8601 280 return dt.toInstant(ZoneOffset.UTC).toString(); 281 } 282 283 private static SearchResult searchName(String area) throws IOException { 284 return searchName(NameFinder.queryNominatim(area)); 285 } 286 287 static SearchResult searchName(List<SearchResult> results) { 288 return results.stream().filter( 289 x -> !OsmPrimitiveType.NODE.equals(x.getOsmId().getType())).iterator().next(); 290 } 291 292 static String geocodeArea(String area) throws IOException { 293 // Offsets defined in https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#By_element_id 294 final EnumMap<OsmPrimitiveType, Long> idOffset = new EnumMap<>(OsmPrimitiveType.class); 295 idOffset.put(OsmPrimitiveType.NODE, 0L); 296 idOffset.put(OsmPrimitiveType.WAY, 2_400_000_000L); 297 idOffset.put(OsmPrimitiveType.RELATION, 3_600_000_000L); 298 final PrimitiveId osmId = searchName(area).getOsmId(); 299 Logging.debug("Area '{0}' resolved to {1}", area, osmId); 300 return String.format("area(%d)", osmId.getUniqueId() + idOffset.get(osmId.getType())); 301 } 302 303 static String geocodeBbox(String area) throws IOException { 304 Bounds bounds = searchName(area).getBounds(); 305 return bounds.getMinLat() + "," + bounds.getMinLon() + "," + bounds.getMaxLat() + "," + bounds.getMaxLon(); 306 } 307 308 static String geocodeCoords(String area) throws IOException { 309 SearchResult result = searchName(area); 310 return result.getLat() + "," + result.getLon(); 311 } 312 313 static String geocodeId(String area) throws IOException { 314 PrimitiveId osmId = searchName(area).getOsmId(); 315 return String.format("%s(%d)", osmId.getType().getAPIName(), osmId.getUniqueId()); 316 } 317 318 @Override 319 protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason, 320 boolean uncompressAccordingToContentDisposition) throws OsmTransferException { 321 try { 322 int index = urlStr.indexOf(DATA_PREFIX); 323 // Make an HTTP POST request instead of a simple GET, allows more complex queries 324 return super.getInputStreamRaw(urlStr.substring(0, index), 325 progressMonitor, reason, uncompressAccordingToContentDisposition, 326 "POST", Utils.decodeUrl(urlStr.substring(index + DATA_PREFIX.length())).getBytes(StandardCharsets.UTF_8)); 327 } catch (OsmApiException ex) { 328 final String errorIndicator = "Error</strong>: "; 329 if (ex.getMessage() != null && ex.getMessage().contains(errorIndicator)) { 330 final String errorPlusRest = ex.getMessage().split(errorIndicator)[1]; 331 if (errorPlusRest != null) { 332 ex.setErrorHeader(errorPlusRest.split("</")[0].replaceAll(".*::request_read_and_idx::", "")); 333 } 334 } 335 throw ex; 336 } 337 } 338 339 @Override 340 protected void adaptRequest(HttpClient request) { 341 // see https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#timeout 342 final Matcher timeoutMatcher = Pattern.compile("\\[timeout:(\\d+)\\]").matcher(overpassQuery); 343 final int timeout; 344 if (timeoutMatcher.find()) { 345 timeout = (int) TimeUnit.SECONDS.toMillis(Integer.parseInt(timeoutMatcher.group(1))); 346 } else { 347 timeout = (int) TimeUnit.MINUTES.toMillis(3); 348 } 349 request.setConnectTimeout(timeout); 350 request.setReadTimeout(timeout); 351 } 352 353 @Override 354 protected String getTaskName() { 355 return tr("Contacting Server..."); 356 } 357 358 @Override 359 protected DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException { 360 AbstractReader reader = null; 361 Matcher m = OUTPUT_FORMAT_STATEMENT.matcher(overpassQuery); 362 if (m.matches()) { 363 Class<? extends AbstractReader> readerClass = outputFormatReaders.get(OverpassOutpoutFormat.from(m.group(1))); 364 if (readerClass != null) { 365 try { 366 reader = readerClass.getDeclaredConstructor().newInstance(); 367 } catch (ReflectiveOperationException | IllegalArgumentException | SecurityException e) { 368 Logging.error(e); 369 } 370 } 371 } 372 if (reader == null) { 373 reader = new OverpassOsmReader(); 374 } 375 return reader.doParseDataSet(source, progressMonitor); 376 } 377 378 @Override 379 public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException { 380 381 DataSet ds = super.parseOsm(progressMonitor); 382 383 // add bounds if necessary (note that Overpass API does not return bounds in the response XML) 384 if (ds != null && ds.getDataSources().isEmpty() && overpassQuery.contains("{{bbox}}")) { 385 if (crosses180th) { 386 Bounds bounds = new Bounds(lat1, lon1, lat2, 180.0); 387 DataSource src = new DataSource(bounds, getBaseUrl()); 388 ds.addDataSource(src); 389 390 bounds = new Bounds(lat1, -180.0, lat2, lon2); 391 src = new DataSource(bounds, getBaseUrl()); 392 ds.addDataSource(src); 393 } else { 394 Bounds bounds = new Bounds(lat1, lon1, lat2, lon2); 395 DataSource src = new DataSource(bounds, getBaseUrl()); 396 ds.addDataSource(src); 397 } 398 } 399 400 return ds; 401 } 402 403 /** 404 * Fixes Overpass API query to make sure it will be accepted by JOSM. 405 * @param query Overpass query to check 406 * @return fixed query 407 * @since 13335 408 */ 409 public static String fixQuery(String query) { 410 return query == null ? query : query 411 .replaceAll("out( body| skel| ids)?( id| qt)?;", "out meta$2;") 412 .replaceAll("(?s)\\[out:(json|csv)[^\\]]*\\]", "[out:xml]"); 413 } 414}