001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.data.validation.tests; 003 004import static org.openstreetmap.josm.tools.I18n.marktr; 005import static org.openstreetmap.josm.tools.I18n.tr; 006 007import java.awt.GridBagConstraints; 008import java.awt.event.ActionListener; 009import java.io.BufferedReader; 010import java.io.IOException; 011import java.util.ArrayList; 012import java.util.Arrays; 013import java.util.Collection; 014import java.util.HashMap; 015import java.util.List; 016import java.util.Locale; 017import java.util.Map; 018import java.util.Map.Entry; 019import java.util.Set; 020import java.util.regex.Matcher; 021import java.util.regex.Pattern; 022import java.util.regex.PatternSyntaxException; 023 024import javax.swing.JCheckBox; 025import javax.swing.JLabel; 026import javax.swing.JPanel; 027 028import org.openstreetmap.josm.command.ChangePropertyCommand; 029import org.openstreetmap.josm.command.ChangePropertyKeyCommand; 030import org.openstreetmap.josm.command.Command; 031import org.openstreetmap.josm.command.SequenceCommand; 032import org.openstreetmap.josm.data.osm.AbstractPrimitive; 033import org.openstreetmap.josm.data.osm.OsmPrimitive; 034import org.openstreetmap.josm.data.osm.OsmPrimitiveType; 035import org.openstreetmap.josm.data.osm.OsmUtils; 036import org.openstreetmap.josm.data.osm.Tag; 037import org.openstreetmap.josm.data.osm.Tagged; 038import org.openstreetmap.josm.data.preferences.sources.ValidatorPrefHelper; 039import org.openstreetmap.josm.data.validation.Severity; 040import org.openstreetmap.josm.data.validation.Test.TagTest; 041import org.openstreetmap.josm.data.validation.TestError; 042import org.openstreetmap.josm.data.validation.util.Entities; 043import org.openstreetmap.josm.gui.progress.ProgressMonitor; 044import org.openstreetmap.josm.gui.tagging.presets.TaggingPreset; 045import org.openstreetmap.josm.gui.tagging.presets.TaggingPresetItem; 046import org.openstreetmap.josm.gui.tagging.presets.TaggingPresets; 047import org.openstreetmap.josm.gui.tagging.presets.items.Check; 048import org.openstreetmap.josm.gui.tagging.presets.items.CheckGroup; 049import org.openstreetmap.josm.gui.tagging.presets.items.KeyedItem; 050import org.openstreetmap.josm.gui.widgets.EditableList; 051import org.openstreetmap.josm.io.CachedFile; 052import org.openstreetmap.josm.spi.preferences.Config; 053import org.openstreetmap.josm.tools.GBC; 054import org.openstreetmap.josm.tools.Logging; 055import org.openstreetmap.josm.tools.MultiMap; 056import org.openstreetmap.josm.tools.Utils; 057 058/** 059 * Check for misspelled or wrong tags 060 * 061 * @author frsantos 062 * @since 3669 063 */ 064public class TagChecker extends TagTest { 065 066 /** The config file of ignored tags */ 067 public static final String IGNORE_FILE = "resource://data/validator/ignoretags.cfg"; 068 /** The config file of dictionary words */ 069 public static final String SPELL_FILE = "resource://data/validator/words.cfg"; 070 071 /** Normalized keys: the key should be substituted by the value if the key was not found in presets */ 072 private static final Map<String, String> harmonizedKeys = new HashMap<>(); 073 /** The spell check preset values which are not stored in TaggingPresets */ 074 private static volatile MultiMap<String, String> additionalPresetsValueData; 075 /** The TagChecker data */ 076 private static final List<CheckerData> checkerData = new ArrayList<>(); 077 private static final List<String> ignoreDataStartsWith = new ArrayList<>(); 078 private static final List<String> ignoreDataEquals = new ArrayList<>(); 079 private static final List<String> ignoreDataEndsWith = new ArrayList<>(); 080 private static final List<Tag> ignoreDataTag = new ArrayList<>(); 081 082 /** The preferences prefix */ 083 protected static final String PREFIX = ValidatorPrefHelper.PREFIX + "." + TagChecker.class.getSimpleName(); 084 085 /** 086 * The preference key to check values 087 */ 088 public static final String PREF_CHECK_VALUES = PREFIX + ".checkValues"; 089 /** 090 * The preference key to check keys 091 */ 092 public static final String PREF_CHECK_KEYS = PREFIX + ".checkKeys"; 093 /** 094 * The preference key to enable complex checks 095 */ 096 public static final String PREF_CHECK_COMPLEX = PREFIX + ".checkComplex"; 097 /** 098 * The preference key to search for fixme tags 099 */ 100 public static final String PREF_CHECK_FIXMES = PREFIX + ".checkFixmes"; 101 102 /** 103 * The preference key for source files 104 * @see #DEFAULT_SOURCES 105 */ 106 public static final String PREF_SOURCES = PREFIX + ".source"; 107 108 /** 109 * The preference key to check keys - used before upload 110 */ 111 public static final String PREF_CHECK_KEYS_BEFORE_UPLOAD = PREF_CHECK_KEYS + "BeforeUpload"; 112 /** 113 * The preference key to check values - used before upload 114 */ 115 public static final String PREF_CHECK_VALUES_BEFORE_UPLOAD = PREF_CHECK_VALUES + "BeforeUpload"; 116 /** 117 * The preference key to run complex tests - used before upload 118 */ 119 public static final String PREF_CHECK_COMPLEX_BEFORE_UPLOAD = PREF_CHECK_COMPLEX + "BeforeUpload"; 120 /** 121 * The preference key to search for fixmes - used before upload 122 */ 123 public static final String PREF_CHECK_FIXMES_BEFORE_UPLOAD = PREF_CHECK_FIXMES + "BeforeUpload"; 124 125 protected boolean checkKeys; 126 protected boolean checkValues; 127 protected boolean checkComplex; 128 protected boolean checkFixmes; 129 130 protected JCheckBox prefCheckKeys; 131 protected JCheckBox prefCheckValues; 132 protected JCheckBox prefCheckComplex; 133 protected JCheckBox prefCheckFixmes; 134 protected JCheckBox prefCheckPaint; 135 136 protected JCheckBox prefCheckKeysBeforeUpload; 137 protected JCheckBox prefCheckValuesBeforeUpload; 138 protected JCheckBox prefCheckComplexBeforeUpload; 139 protected JCheckBox prefCheckFixmesBeforeUpload; 140 protected JCheckBox prefCheckPaintBeforeUpload; 141 142 // CHECKSTYLE.OFF: SingleSpaceSeparator 143 protected static final int EMPTY_VALUES = 1200; 144 protected static final int INVALID_KEY = 1201; 145 protected static final int INVALID_VALUE = 1202; 146 protected static final int FIXME = 1203; 147 protected static final int INVALID_SPACE = 1204; 148 protected static final int INVALID_KEY_SPACE = 1205; 149 protected static final int INVALID_HTML = 1206; /* 1207 was PAINT */ 150 protected static final int LONG_VALUE = 1208; 151 protected static final int LONG_KEY = 1209; 152 protected static final int LOW_CHAR_VALUE = 1210; 153 protected static final int LOW_CHAR_KEY = 1211; 154 protected static final int MISSPELLED_VALUE = 1212; 155 protected static final int MISSPELLED_KEY = 1213; 156 protected static final int MULTIPLE_SPACES = 1214; 157 // CHECKSTYLE.ON: SingleSpaceSeparator 158 // 1250 and up is used by tagcheck 159 160 protected EditableList sourcesList; 161 162 private static final List<String> DEFAULT_SOURCES = Arrays.asList(/*DATA_FILE, */IGNORE_FILE, SPELL_FILE); 163 164 /** 165 * Constructor 166 */ 167 public TagChecker() { 168 super(tr("Tag checker"), tr("This test checks for errors in tag keys and values.")); 169 } 170 171 @Override 172 public void initialize() throws IOException { 173 initializeData(); 174 initializePresets(); 175 } 176 177 /** 178 * Reads the spellcheck file into a HashMap. 179 * The data file is a list of words, beginning with +/-. If it starts with +, 180 * the word is valid, but if it starts with -, the word should be replaced 181 * by the nearest + word before this. 182 * 183 * @throws IOException if any I/O error occurs 184 */ 185 private static void initializeData() throws IOException { 186 checkerData.clear(); 187 ignoreDataStartsWith.clear(); 188 ignoreDataEquals.clear(); 189 ignoreDataEndsWith.clear(); 190 ignoreDataTag.clear(); 191 harmonizedKeys.clear(); 192 193 StringBuilder errorSources = new StringBuilder(); 194 for (String source : Config.getPref().getList(PREF_SOURCES, DEFAULT_SOURCES)) { 195 try ( 196 CachedFile cf = new CachedFile(source); 197 BufferedReader reader = cf.getContentReader() 198 ) { 199 String okValue = null; 200 boolean tagcheckerfile = false; 201 boolean ignorefile = false; 202 boolean isFirstLine = true; 203 String line; 204 while ((line = reader.readLine()) != null && (tagcheckerfile || !line.isEmpty())) { 205 if (line.startsWith("#")) { 206 if (line.startsWith("# JOSM TagChecker")) { 207 tagcheckerfile = true; 208 if (!DEFAULT_SOURCES.contains(source)) { 209 Logging.info(tr("Adding {0} to tag checker", source)); 210 } 211 } else 212 if (line.startsWith("# JOSM IgnoreTags")) { 213 ignorefile = true; 214 if (!DEFAULT_SOURCES.contains(source)) { 215 Logging.info(tr("Adding {0} to ignore tags", source)); 216 } 217 } 218 } else if (ignorefile) { 219 line = line.trim(); 220 if (line.length() < 4) { 221 continue; 222 } 223 224 String key = line.substring(0, 2); 225 line = line.substring(2); 226 227 switch (key) { 228 case "S:": 229 ignoreDataStartsWith.add(line); 230 break; 231 case "E:": 232 ignoreDataEquals.add(line); 233 break; 234 case "F:": 235 ignoreDataEndsWith.add(line); 236 break; 237 case "K:": 238 ignoreDataTag.add(Tag.ofString(line)); 239 break; 240 default: 241 if (!key.startsWith(";")) { 242 Logging.warn("Unsupported TagChecker key: " + key); 243 } 244 } 245 } else if (tagcheckerfile) { 246 if (!line.isEmpty()) { 247 CheckerData d = new CheckerData(); 248 String err = d.getData(line); 249 250 if (err == null) { 251 checkerData.add(d); 252 } else { 253 Logging.error(tr("Invalid tagchecker line - {0}: {1}", err, line)); 254 } 255 } 256 } else if (line.charAt(0) == '+') { 257 okValue = line.substring(1); 258 } else if (line.charAt(0) == '-' && okValue != null) { 259 harmonizedKeys.put(harmonizeKey(line.substring(1)), okValue); 260 } else { 261 Logging.error(tr("Invalid spellcheck line: {0}", line)); 262 } 263 if (isFirstLine) { 264 isFirstLine = false; 265 if (!(tagcheckerfile || ignorefile) && !DEFAULT_SOURCES.contains(source)) { 266 Logging.info(tr("Adding {0} to spellchecker", source)); 267 } 268 } 269 } 270 } catch (IOException e) { 271 Logging.error(e); 272 errorSources.append(source).append('\n'); 273 } 274 } 275 276 if (errorSources.length() > 0) 277 throw new IOException(tr("Could not access data file(s):\n{0}", errorSources)); 278 } 279 280 /** 281 * Reads the presets data. 282 * 283 */ 284 public static void initializePresets() { 285 286 if (!Config.getPref().getBoolean(PREF_CHECK_VALUES, true)) 287 return; 288 289 Collection<TaggingPreset> presets = TaggingPresets.getTaggingPresets(); 290 if (!presets.isEmpty()) { 291 additionalPresetsValueData = new MultiMap<>(); 292 for (String a : AbstractPrimitive.getUninterestingKeys()) { 293 additionalPresetsValueData.putVoid(a); 294 } 295 // TODO directionKeys are no longer in OsmPrimitive (search pattern is used instead) 296 for (String a : Config.getPref().getList(ValidatorPrefHelper.PREFIX + ".knownkeys", 297 Arrays.asList("is_in", "int_ref", "fixme", "population"))) { 298 additionalPresetsValueData.putVoid(a); 299 } 300 for (TaggingPreset p : presets) { 301 for (TaggingPresetItem i : p.data) { 302 if (i instanceof KeyedItem) { 303 addPresetValue((KeyedItem) i); 304 } else if (i instanceof CheckGroup) { 305 for (Check c : ((CheckGroup) i).checks) { 306 addPresetValue(c); 307 } 308 } 309 } 310 } 311 } 312 } 313 314 private static void addPresetValue(KeyedItem ky) { 315 Collection<String> values = ky.getValues(); 316 if (ky.key != null && values != null) { 317 harmonizedKeys.put(harmonizeKey(ky.key), ky.key); 318 } 319 } 320 321 /** 322 * Checks given string (key or value) if it contains characters with code below 0x20 (either newline or some other special characters) 323 * @param s string to check 324 * @return {@code true} if {@code s} contains characters with code below 0x20 325 */ 326 private static boolean containsLow(String s) { 327 if (s == null) 328 return false; 329 for (int i = 0; i < s.length(); i++) { 330 if (s.charAt(i) < 0x20) 331 return true; 332 } 333 return false; 334 } 335 336 private static Set<String> getPresetValues(String key) { 337 Set<String> res = TaggingPresets.getPresetValues(key); 338 if (res != null) 339 return res; 340 return additionalPresetsValueData.get(key); 341 } 342 343 /** 344 * Determines if the given key is in internal presets. 345 * @param key key 346 * @return {@code true} if the given key is in internal presets 347 * @since 9023 348 */ 349 public static boolean isKeyInPresets(String key) { 350 return getPresetValues(key) != null; 351 } 352 353 /** 354 * Determines if the given tag is in internal presets. 355 * @param key key 356 * @param value value 357 * @return {@code true} if the given tag is in internal presets 358 * @since 9023 359 */ 360 public static boolean isTagInPresets(String key, String value) { 361 final Set<String> values = getPresetValues(key); 362 return values != null && (values.isEmpty() || values.contains(value)); 363 } 364 365 /** 366 * Returns the list of ignored tags. 367 * @return the list of ignored tags 368 * @since 9023 369 */ 370 public static List<Tag> getIgnoredTags() { 371 return new ArrayList<>(ignoreDataTag); 372 } 373 374 /** 375 * Determines if the given tag is ignored for checks "key/tag not in presets". 376 * @param key key 377 * @param value value 378 * @return {@code true} if the given tag is ignored 379 * @since 9023 380 */ 381 public static boolean isTagIgnored(String key, String value) { 382 boolean tagInPresets = isTagInPresets(key, value); 383 boolean ignore = false; 384 385 for (String a : ignoreDataStartsWith) { 386 if (key.startsWith(a)) { 387 ignore = true; 388 } 389 } 390 for (String a : ignoreDataEquals) { 391 if (key.equals(a)) { 392 ignore = true; 393 } 394 } 395 for (String a : ignoreDataEndsWith) { 396 if (key.endsWith(a)) { 397 ignore = true; 398 } 399 } 400 401 if (!tagInPresets) { 402 for (Tag a : ignoreDataTag) { 403 if (key.equals(a.getKey()) && value.equals(a.getValue())) { 404 ignore = true; 405 } 406 } 407 } 408 return ignore; 409 } 410 411 /** 412 * Checks the primitive tags 413 * @param p The primitive to check 414 */ 415 @Override 416 public void check(OsmPrimitive p) { 417 // Just a collection to know if a primitive has been already marked with error 418 MultiMap<OsmPrimitive, String> withErrors = new MultiMap<>(); 419 420 if (checkComplex) { 421 Map<String, String> keys = p.getKeys(); 422 for (CheckerData d : checkerData) { 423 if (d.match(p, keys)) { 424 errors.add(TestError.builder(this, d.getSeverity(), d.getCode()) 425 .message(tr("Suspicious tag/value combinations"), d.getDescription()) 426 .primitives(p) 427 .build()); 428 withErrors.put(p, "TC"); 429 } 430 } 431 } 432 433 for (Entry<String, String> prop : p.getKeys().entrySet()) { 434 String s = marktr("Tag ''{0}'' invalid."); 435 String key = prop.getKey(); 436 String value = prop.getValue(); 437 if (checkValues && (containsLow(value)) && !withErrors.contains(p, "ICV")) { 438 errors.add(TestError.builder(this, Severity.WARNING, LOW_CHAR_VALUE) 439 .message(tr("Tag value contains character with code less than 0x20"), s, key) 440 .primitives(p) 441 .build()); 442 withErrors.put(p, "ICV"); 443 } 444 if (checkKeys && (containsLow(key)) && !withErrors.contains(p, "ICK")) { 445 errors.add(TestError.builder(this, Severity.WARNING, LOW_CHAR_KEY) 446 .message(tr("Tag key contains character with code less than 0x20"), s, key) 447 .primitives(p) 448 .build()); 449 withErrors.put(p, "ICK"); 450 } 451 if (checkValues && (value != null && value.length() > Tagged.MAX_TAG_LENGTH) && !withErrors.contains(p, "LV")) { 452 errors.add(TestError.builder(this, Severity.ERROR, LONG_VALUE) 453 .message(tr("Tag value longer than {0} characters ({1} characters)", Tagged.MAX_TAG_LENGTH, value.length()), s, key) 454 .primitives(p) 455 .build()); 456 withErrors.put(p, "LV"); 457 } 458 if (checkKeys && (key != null && key.length() > Tagged.MAX_TAG_LENGTH) && !withErrors.contains(p, "LK")) { 459 errors.add(TestError.builder(this, Severity.ERROR, LONG_KEY) 460 .message(tr("Tag key longer than {0} characters ({1} characters)", Tagged.MAX_TAG_LENGTH, key.length()), s, key) 461 .primitives(p) 462 .build()); 463 withErrors.put(p, "LK"); 464 } 465 if (checkValues && (value == null || value.trim().isEmpty()) && !withErrors.contains(p, "EV")) { 466 errors.add(TestError.builder(this, Severity.WARNING, EMPTY_VALUES) 467 .message(tr("Tags with empty values"), s, key) 468 .primitives(p) 469 .build()); 470 withErrors.put(p, "EV"); 471 } 472 if (checkKeys && key != null && key.indexOf(' ') >= 0 && !withErrors.contains(p, "IPK")) { 473 errors.add(TestError.builder(this, Severity.WARNING, INVALID_KEY_SPACE) 474 .message(tr("Invalid white space in property key"), s, key) 475 .primitives(p) 476 .build()); 477 withErrors.put(p, "IPK"); 478 } 479 if (checkValues && value != null && (value.startsWith(" ") || value.endsWith(" ")) && !withErrors.contains(p, "SPACE")) { 480 errors.add(TestError.builder(this, Severity.WARNING, INVALID_SPACE) 481 .message(tr("Property values start or end with white space"), s, key) 482 .primitives(p) 483 .build()); 484 withErrors.put(p, "SPACE"); 485 } 486 if (checkValues && value != null && value.contains(" ") && !withErrors.contains(p, "SPACE")) { 487 errors.add(TestError.builder(this, Severity.WARNING, MULTIPLE_SPACES) 488 .message(tr("Property values contain multiple white spaces"), s, key) 489 .primitives(p) 490 .build()); 491 withErrors.put(p, "SPACE"); 492 } 493 if (checkValues && value != null && !value.equals(Entities.unescape(value)) && !withErrors.contains(p, "HTML")) { 494 errors.add(TestError.builder(this, Severity.OTHER, INVALID_HTML) 495 .message(tr("Property values contain HTML entity"), s, key) 496 .primitives(p) 497 .build()); 498 withErrors.put(p, "HTML"); 499 } 500 if (checkValues && key != null && value != null && !value.isEmpty() && additionalPresetsValueData != null 501 && !isTagIgnored(key, value)) { 502 if (!isKeyInPresets(key)) { 503 String prettifiedKey = harmonizeKey(key); 504 String fixedKey = harmonizedKeys.get(prettifiedKey); 505 if (fixedKey != null && !"".equals(fixedKey) && !fixedKey.equals(key)) { 506 // misspelled preset key 507 final TestError.Builder error = TestError.builder(this, Severity.WARNING, MISSPELLED_KEY) 508 .message(tr("Misspelled property key"), marktr("Key ''{0}'' looks like ''{1}''."), key, fixedKey) 509 .primitives(p); 510 if (p.hasKey(fixedKey)) { 511 errors.add(error.build()); 512 } else { 513 errors.add(error.fix(() -> new ChangePropertyKeyCommand(p, key, fixedKey)).build()); 514 } 515 withErrors.put(p, "WPK"); 516 } else { 517 errors.add(TestError.builder(this, Severity.OTHER, INVALID_VALUE) 518 .message(tr("Presets do not contain property key"), marktr("Key ''{0}'' not in presets."), key) 519 .primitives(p) 520 .build()); 521 withErrors.put(p, "UPK"); 522 } 523 } else if (!isTagInPresets(key, value)) { 524 // try to fix common typos and check again if value is still unknown 525 String fixedValue = harmonizeValue(prop.getValue()); 526 Map<String, String> possibleValues = getPossibleValues(getPresetValues(key)); 527 if (possibleValues.containsKey(fixedValue)) { 528 final String newValue = possibleValues.get(fixedValue); 529 // misspelled preset value 530 errors.add(TestError.builder(this, Severity.WARNING, MISSPELLED_VALUE) 531 .message(tr("Misspelled property value"), 532 marktr("Value ''{0}'' for key ''{1}'' looks like ''{2}''."), prop.getValue(), key, newValue) 533 .primitives(p) 534 .fix(() -> new ChangePropertyCommand(p, key, newValue)) 535 .build()); 536 withErrors.put(p, "WPV"); 537 } else { 538 // unknown preset value 539 errors.add(TestError.builder(this, Severity.OTHER, INVALID_VALUE) 540 .message(tr("Presets do not contain property value"), 541 marktr("Value ''{0}'' for key ''{1}'' not in presets."), prop.getValue(), key) 542 .primitives(p) 543 .build()); 544 withErrors.put(p, "UPV"); 545 } 546 } 547 } 548 if (checkFixmes && key != null && value != null && !value.isEmpty() && isFixme(key, value) && !withErrors.contains(p, "FIXME")) { 549 errors.add(TestError.builder(this, Severity.OTHER, FIXME) 550 .message(tr("FIXMES")) 551 .primitives(p) 552 .build()); 553 withErrors.put(p, "FIXME"); 554 } 555 } 556 } 557 558 private static boolean isFixme(String key, String value) { 559 return key.toLowerCase(Locale.ENGLISH).contains("fixme") || key.contains("todo") 560 || value.toLowerCase(Locale.ENGLISH).contains("fixme") || value.contains("check and delete"); 561 } 562 563 private static Map<String, String> getPossibleValues(Set<String> values) { 564 // generate a map with common typos 565 Map<String, String> map = new HashMap<>(); 566 if (values != null) { 567 for (String value : values) { 568 map.put(value, value); 569 if (value.contains("_")) { 570 map.put(value.replace("_", ""), value); 571 } 572 } 573 } 574 return map; 575 } 576 577 private static String harmonizeKey(String key) { 578 return Utils.strip(key.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(':', '_').replace(' ', '_'), "-_;:,"); 579 } 580 581 private static String harmonizeValue(String value) { 582 return Utils.strip(value.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(' ', '_'), "-_;:,"); 583 } 584 585 @Override 586 public void startTest(ProgressMonitor monitor) { 587 super.startTest(monitor); 588 checkKeys = Config.getPref().getBoolean(PREF_CHECK_KEYS, true); 589 if (isBeforeUpload) { 590 checkKeys = checkKeys && Config.getPref().getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true); 591 } 592 593 checkValues = Config.getPref().getBoolean(PREF_CHECK_VALUES, true); 594 if (isBeforeUpload) { 595 checkValues = checkValues && Config.getPref().getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true); 596 } 597 598 checkComplex = Config.getPref().getBoolean(PREF_CHECK_COMPLEX, true); 599 if (isBeforeUpload) { 600 checkComplex = checkComplex && Config.getPref().getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true); 601 } 602 603 checkFixmes = Config.getPref().getBoolean(PREF_CHECK_FIXMES, true); 604 if (isBeforeUpload) { 605 checkFixmes = checkFixmes && Config.getPref().getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true); 606 } 607 } 608 609 @Override 610 public void visit(Collection<OsmPrimitive> selection) { 611 if (checkKeys || checkValues || checkComplex || checkFixmes) { 612 super.visit(selection); 613 } 614 } 615 616 @Override 617 public void addGui(JPanel testPanel) { 618 GBC a = GBC.eol(); 619 a.anchor = GridBagConstraints.EAST; 620 621 testPanel.add(new JLabel(name+" :"), GBC.eol().insets(3, 0, 0, 0)); 622 623 prefCheckKeys = new JCheckBox(tr("Check property keys."), Config.getPref().getBoolean(PREF_CHECK_KEYS, true)); 624 prefCheckKeys.setToolTipText(tr("Validate that property keys are valid checking against list of words.")); 625 testPanel.add(prefCheckKeys, GBC.std().insets(20, 0, 0, 0)); 626 627 prefCheckKeysBeforeUpload = new JCheckBox(); 628 prefCheckKeysBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true)); 629 testPanel.add(prefCheckKeysBeforeUpload, a); 630 631 prefCheckComplex = new JCheckBox(tr("Use complex property checker."), Config.getPref().getBoolean(PREF_CHECK_COMPLEX, true)); 632 prefCheckComplex.setToolTipText(tr("Validate property values and tags using complex rules.")); 633 testPanel.add(prefCheckComplex, GBC.std().insets(20, 0, 0, 0)); 634 635 prefCheckComplexBeforeUpload = new JCheckBox(); 636 prefCheckComplexBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true)); 637 testPanel.add(prefCheckComplexBeforeUpload, a); 638 639 final Collection<String> sources = Config.getPref().getList(PREF_SOURCES, DEFAULT_SOURCES); 640 sourcesList = new EditableList(tr("TagChecker source")); 641 sourcesList.setItems(sources); 642 testPanel.add(new JLabel(tr("Data sources ({0})", "*.cfg")), GBC.eol().insets(23, 0, 0, 0)); 643 testPanel.add(sourcesList, GBC.eol().fill(GridBagConstraints.HORIZONTAL).insets(23, 0, 0, 0)); 644 645 ActionListener disableCheckActionListener = e -> handlePrefEnable(); 646 prefCheckKeys.addActionListener(disableCheckActionListener); 647 prefCheckKeysBeforeUpload.addActionListener(disableCheckActionListener); 648 prefCheckComplex.addActionListener(disableCheckActionListener); 649 prefCheckComplexBeforeUpload.addActionListener(disableCheckActionListener); 650 651 handlePrefEnable(); 652 653 prefCheckValues = new JCheckBox(tr("Check property values."), Config.getPref().getBoolean(PREF_CHECK_VALUES, true)); 654 prefCheckValues.setToolTipText(tr("Validate that property values are valid checking against presets.")); 655 testPanel.add(prefCheckValues, GBC.std().insets(20, 0, 0, 0)); 656 657 prefCheckValuesBeforeUpload = new JCheckBox(); 658 prefCheckValuesBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true)); 659 testPanel.add(prefCheckValuesBeforeUpload, a); 660 661 prefCheckFixmes = new JCheckBox(tr("Check for FIXMES."), Config.getPref().getBoolean(PREF_CHECK_FIXMES, true)); 662 prefCheckFixmes.setToolTipText(tr("Looks for nodes or ways with FIXME in any property value.")); 663 testPanel.add(prefCheckFixmes, GBC.std().insets(20, 0, 0, 0)); 664 665 prefCheckFixmesBeforeUpload = new JCheckBox(); 666 prefCheckFixmesBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true)); 667 testPanel.add(prefCheckFixmesBeforeUpload, a); 668 } 669 670 /** 671 * Enables/disables the source list field 672 */ 673 public void handlePrefEnable() { 674 boolean selected = prefCheckKeys.isSelected() || prefCheckKeysBeforeUpload.isSelected() 675 || prefCheckComplex.isSelected() || prefCheckComplexBeforeUpload.isSelected(); 676 sourcesList.setEnabled(selected); 677 } 678 679 @Override 680 public boolean ok() { 681 enabled = prefCheckKeys.isSelected() || prefCheckValues.isSelected() || prefCheckComplex.isSelected() || prefCheckFixmes.isSelected(); 682 testBeforeUpload = prefCheckKeysBeforeUpload.isSelected() || prefCheckValuesBeforeUpload.isSelected() 683 || prefCheckFixmesBeforeUpload.isSelected() || prefCheckComplexBeforeUpload.isSelected(); 684 685 Config.getPref().putBoolean(PREF_CHECK_VALUES, prefCheckValues.isSelected()); 686 Config.getPref().putBoolean(PREF_CHECK_COMPLEX, prefCheckComplex.isSelected()); 687 Config.getPref().putBoolean(PREF_CHECK_KEYS, prefCheckKeys.isSelected()); 688 Config.getPref().putBoolean(PREF_CHECK_FIXMES, prefCheckFixmes.isSelected()); 689 Config.getPref().putBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, prefCheckValuesBeforeUpload.isSelected()); 690 Config.getPref().putBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, prefCheckComplexBeforeUpload.isSelected()); 691 Config.getPref().putBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, prefCheckKeysBeforeUpload.isSelected()); 692 Config.getPref().putBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, prefCheckFixmesBeforeUpload.isSelected()); 693 return Config.getPref().putList(PREF_SOURCES, sourcesList.getItems()); 694 } 695 696 @Override 697 public Command fixError(TestError testError) { 698 List<Command> commands = new ArrayList<>(50); 699 700 Collection<? extends OsmPrimitive> primitives = testError.getPrimitives(); 701 for (OsmPrimitive p : primitives) { 702 Map<String, String> tags = p.getKeys(); 703 if (tags.isEmpty()) { 704 continue; 705 } 706 707 for (Entry<String, String> prop: tags.entrySet()) { 708 String key = prop.getKey(); 709 String value = prop.getValue(); 710 if (value == null || value.trim().isEmpty()) { 711 commands.add(new ChangePropertyCommand(p, key, null)); 712 } else if (value.startsWith(" ") || value.endsWith(" ") || value.contains(" ")) { 713 commands.add(new ChangePropertyCommand(p, key, Utils.removeWhiteSpaces(value))); 714 } else if (key.startsWith(" ") || key.endsWith(" ") || key.contains(" ")) { 715 commands.add(new ChangePropertyKeyCommand(p, key, Utils.removeWhiteSpaces(key))); 716 } else { 717 String evalue = Entities.unescape(value); 718 if (!evalue.equals(value)) { 719 commands.add(new ChangePropertyCommand(p, key, evalue)); 720 } 721 } 722 } 723 } 724 725 if (commands.isEmpty()) 726 return null; 727 if (commands.size() == 1) 728 return commands.get(0); 729 730 return new SequenceCommand(tr("Fix tags"), commands); 731 } 732 733 @Override 734 public boolean isFixable(TestError testError) { 735 if (testError.getTester() instanceof TagChecker) { 736 int code = testError.getCode(); 737 return code == INVALID_KEY || code == EMPTY_VALUES || code == INVALID_SPACE || 738 code == INVALID_KEY_SPACE || code == INVALID_HTML || code == MISSPELLED_VALUE || 739 code == MULTIPLE_SPACES; 740 } 741 742 return false; 743 } 744 745 protected static class CheckerData { 746 private String description; 747 protected List<CheckerElement> data = new ArrayList<>(); 748 private OsmPrimitiveType type; 749 private TagCheckLevel level; 750 protected Severity severity; 751 752 private enum TagCheckLevel { 753 TAG_CHECK_ERROR(1250), 754 TAG_CHECK_WARN(1260), 755 TAG_CHECK_INFO(1270); 756 757 final int code; 758 759 TagCheckLevel(int code) { 760 this.code = code; 761 } 762 } 763 764 protected static class CheckerElement { 765 public Object tag; 766 public Object value; 767 public boolean noMatch; 768 public boolean tagAll; 769 public boolean valueAll; 770 public boolean valueBool; 771 772 private static Pattern getPattern(String str) { 773 if (str.endsWith("/i")) 774 return Pattern.compile(str.substring(1, str.length()-2), Pattern.CASE_INSENSITIVE); 775 if (str.endsWith("/")) 776 return Pattern.compile(str.substring(1, str.length()-1)); 777 778 throw new IllegalStateException(); 779 } 780 781 public CheckerElement(String exp) { 782 Matcher m = Pattern.compile("(.+)([!=]=)(.+)").matcher(exp); 783 m.matches(); 784 785 String n = m.group(1).trim(); 786 787 if ("*".equals(n)) { 788 tagAll = true; 789 } else { 790 tag = n.startsWith("/") ? getPattern(n) : n; 791 noMatch = "!=".equals(m.group(2)); 792 n = m.group(3).trim(); 793 if ("*".equals(n)) { 794 valueAll = true; 795 } else if ("BOOLEAN_TRUE".equals(n)) { 796 valueBool = true; 797 value = OsmUtils.TRUE_VALUE; 798 } else if ("BOOLEAN_FALSE".equals(n)) { 799 valueBool = true; 800 value = OsmUtils.FALSE_VALUE; 801 } else { 802 value = n.startsWith("/") ? getPattern(n) : n; 803 } 804 } 805 } 806 807 public boolean match(Map<String, String> keys) { 808 for (Entry<String, String> prop: keys.entrySet()) { 809 String key = prop.getKey(); 810 String val = valueBool ? OsmUtils.getNamedOsmBoolean(prop.getValue()) : prop.getValue(); 811 if ((tagAll || (tag instanceof Pattern ? ((Pattern) tag).matcher(key).matches() : key.equals(tag))) 812 && (valueAll || (value instanceof Pattern ? ((Pattern) value).matcher(val).matches() : val.equals(value)))) 813 return !noMatch; 814 } 815 return noMatch; 816 } 817 } 818 819 private static final Pattern CLEAN_STR_PATTERN = Pattern.compile(" *# *([^#]+) *$"); 820 private static final Pattern SPLIT_TRIMMED_PATTERN = Pattern.compile(" *: *"); 821 private static final Pattern SPLIT_ELEMENTS_PATTERN = Pattern.compile(" *&& *"); 822 823 public String getData(final String str) { 824 Matcher m = CLEAN_STR_PATTERN.matcher(str); 825 String trimmed = m.replaceFirst("").trim(); 826 try { 827 description = m.group(1); 828 if (description != null && description.isEmpty()) { 829 description = null; 830 } 831 } catch (IllegalStateException e) { 832 Logging.error(e); 833 description = null; 834 } 835 String[] n = SPLIT_TRIMMED_PATTERN.split(trimmed, 3); 836 switch (n[0]) { 837 case "way": 838 type = OsmPrimitiveType.WAY; 839 break; 840 case "node": 841 type = OsmPrimitiveType.NODE; 842 break; 843 case "relation": 844 type = OsmPrimitiveType.RELATION; 845 break; 846 case "*": 847 type = null; 848 break; 849 default: 850 return tr("Could not find element type"); 851 } 852 if (n.length != 3) 853 return tr("Incorrect number of parameters"); 854 855 switch (n[1]) { 856 case "W": 857 severity = Severity.WARNING; 858 level = TagCheckLevel.TAG_CHECK_WARN; 859 break; 860 case "E": 861 severity = Severity.ERROR; 862 level = TagCheckLevel.TAG_CHECK_ERROR; 863 break; 864 case "I": 865 severity = Severity.OTHER; 866 level = TagCheckLevel.TAG_CHECK_INFO; 867 break; 868 default: 869 return tr("Could not find warning level"); 870 } 871 for (String exp: SPLIT_ELEMENTS_PATTERN.split(n[2])) { 872 try { 873 data.add(new CheckerElement(exp)); 874 } catch (IllegalStateException e) { 875 Logging.trace(e); 876 return tr("Illegal expression ''{0}''", exp); 877 } catch (PatternSyntaxException e) { 878 Logging.trace(e); 879 return tr("Illegal regular expression ''{0}''", exp); 880 } 881 } 882 return null; 883 } 884 885 public boolean match(OsmPrimitive osm, Map<String, String> keys) { 886 if (type != null && OsmPrimitiveType.from(osm) != type) 887 return false; 888 889 for (CheckerElement ce : data) { 890 if (!ce.match(keys)) 891 return false; 892 } 893 return true; 894 } 895 896 /** 897 * Returns the error description. 898 * @return the error description 899 */ 900 public String getDescription() { 901 return description; 902 } 903 904 /** 905 * Returns the error severity. 906 * @return the error severity 907 */ 908 public Severity getSeverity() { 909 return severity; 910 } 911 912 /** 913 * Returns the error code. 914 * @return the error code 915 */ 916 public int getCode() { 917 if (type == null) 918 return level.code; 919 920 return level.code + type.ordinal() + 1; 921 } 922 } 923}