001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.data.validation.tests;
003
004import static org.openstreetmap.josm.tools.I18n.marktr;
005import static org.openstreetmap.josm.tools.I18n.tr;
006
007import java.awt.GridBagConstraints;
008import java.awt.event.ActionListener;
009import java.io.BufferedReader;
010import java.io.IOException;
011import java.util.ArrayList;
012import java.util.Arrays;
013import java.util.Collection;
014import java.util.HashMap;
015import java.util.List;
016import java.util.Locale;
017import java.util.Map;
018import java.util.Map.Entry;
019import java.util.Set;
020import java.util.regex.Matcher;
021import java.util.regex.Pattern;
022import java.util.regex.PatternSyntaxException;
023
024import javax.swing.JCheckBox;
025import javax.swing.JLabel;
026import javax.swing.JPanel;
027
028import org.openstreetmap.josm.command.ChangePropertyCommand;
029import org.openstreetmap.josm.command.ChangePropertyKeyCommand;
030import org.openstreetmap.josm.command.Command;
031import org.openstreetmap.josm.command.SequenceCommand;
032import org.openstreetmap.josm.data.osm.AbstractPrimitive;
033import org.openstreetmap.josm.data.osm.OsmPrimitive;
034import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
035import org.openstreetmap.josm.data.osm.OsmUtils;
036import org.openstreetmap.josm.data.osm.Tag;
037import org.openstreetmap.josm.data.osm.Tagged;
038import org.openstreetmap.josm.data.preferences.sources.ValidatorPrefHelper;
039import org.openstreetmap.josm.data.validation.Severity;
040import org.openstreetmap.josm.data.validation.Test.TagTest;
041import org.openstreetmap.josm.data.validation.TestError;
042import org.openstreetmap.josm.data.validation.util.Entities;
043import org.openstreetmap.josm.gui.progress.ProgressMonitor;
044import org.openstreetmap.josm.gui.tagging.presets.TaggingPreset;
045import org.openstreetmap.josm.gui.tagging.presets.TaggingPresetItem;
046import org.openstreetmap.josm.gui.tagging.presets.TaggingPresets;
047import org.openstreetmap.josm.gui.tagging.presets.items.Check;
048import org.openstreetmap.josm.gui.tagging.presets.items.CheckGroup;
049import org.openstreetmap.josm.gui.tagging.presets.items.KeyedItem;
050import org.openstreetmap.josm.gui.widgets.EditableList;
051import org.openstreetmap.josm.io.CachedFile;
052import org.openstreetmap.josm.spi.preferences.Config;
053import org.openstreetmap.josm.tools.GBC;
054import org.openstreetmap.josm.tools.Logging;
055import org.openstreetmap.josm.tools.MultiMap;
056import org.openstreetmap.josm.tools.Utils;
057
058/**
059 * Check for misspelled or wrong tags
060 *
061 * @author frsantos
062 * @since 3669
063 */
064public class TagChecker extends TagTest {
065
066    /** The config file of ignored tags */
067    public static final String IGNORE_FILE = "resource://data/validator/ignoretags.cfg";
068    /** The config file of dictionary words */
069    public static final String SPELL_FILE = "resource://data/validator/words.cfg";
070
071    /** Normalized keys: the key should be substituted by the value if the key was not found in presets */
072    private static final Map<String, String> harmonizedKeys = new HashMap<>();
073    /** The spell check preset values which are not stored in TaggingPresets */
074    private static volatile MultiMap<String, String> additionalPresetsValueData;
075    /** The TagChecker data */
076    private static final List<CheckerData> checkerData = new ArrayList<>();
077    private static final List<String> ignoreDataStartsWith = new ArrayList<>();
078    private static final List<String> ignoreDataEquals = new ArrayList<>();
079    private static final List<String> ignoreDataEndsWith = new ArrayList<>();
080    private static final List<Tag> ignoreDataTag = new ArrayList<>();
081
082    /** The preferences prefix */
083    protected static final String PREFIX = ValidatorPrefHelper.PREFIX + "." + TagChecker.class.getSimpleName();
084
085    /**
086     * The preference key to check values
087     */
088    public static final String PREF_CHECK_VALUES = PREFIX + ".checkValues";
089    /**
090     * The preference key to check keys
091     */
092    public static final String PREF_CHECK_KEYS = PREFIX + ".checkKeys";
093    /**
094     * The preference key to enable complex checks
095     */
096    public static final String PREF_CHECK_COMPLEX = PREFIX + ".checkComplex";
097    /**
098     * The preference key to search for fixme tags
099     */
100    public static final String PREF_CHECK_FIXMES = PREFIX + ".checkFixmes";
101
102    /**
103     * The preference key for source files
104     * @see #DEFAULT_SOURCES
105     */
106    public static final String PREF_SOURCES = PREFIX + ".source";
107
108    /**
109     * The preference key to check keys - used before upload
110     */
111    public static final String PREF_CHECK_KEYS_BEFORE_UPLOAD = PREF_CHECK_KEYS + "BeforeUpload";
112    /**
113     * The preference key to check values - used before upload
114     */
115    public static final String PREF_CHECK_VALUES_BEFORE_UPLOAD = PREF_CHECK_VALUES + "BeforeUpload";
116    /**
117     * The preference key to run complex tests - used before upload
118     */
119    public static final String PREF_CHECK_COMPLEX_BEFORE_UPLOAD = PREF_CHECK_COMPLEX + "BeforeUpload";
120    /**
121     * The preference key to search for fixmes - used before upload
122     */
123    public static final String PREF_CHECK_FIXMES_BEFORE_UPLOAD = PREF_CHECK_FIXMES + "BeforeUpload";
124
125    protected boolean checkKeys;
126    protected boolean checkValues;
127    protected boolean checkComplex;
128    protected boolean checkFixmes;
129
130    protected JCheckBox prefCheckKeys;
131    protected JCheckBox prefCheckValues;
132    protected JCheckBox prefCheckComplex;
133    protected JCheckBox prefCheckFixmes;
134    protected JCheckBox prefCheckPaint;
135
136    protected JCheckBox prefCheckKeysBeforeUpload;
137    protected JCheckBox prefCheckValuesBeforeUpload;
138    protected JCheckBox prefCheckComplexBeforeUpload;
139    protected JCheckBox prefCheckFixmesBeforeUpload;
140    protected JCheckBox prefCheckPaintBeforeUpload;
141
142    // CHECKSTYLE.OFF: SingleSpaceSeparator
143    protected static final int EMPTY_VALUES      = 1200;
144    protected static final int INVALID_KEY       = 1201;
145    protected static final int INVALID_VALUE     = 1202;
146    protected static final int FIXME             = 1203;
147    protected static final int INVALID_SPACE     = 1204;
148    protected static final int INVALID_KEY_SPACE = 1205;
149    protected static final int INVALID_HTML      = 1206; /* 1207 was PAINT */
150    protected static final int LONG_VALUE        = 1208;
151    protected static final int LONG_KEY          = 1209;
152    protected static final int LOW_CHAR_VALUE    = 1210;
153    protected static final int LOW_CHAR_KEY      = 1211;
154    protected static final int MISSPELLED_VALUE  = 1212;
155    protected static final int MISSPELLED_KEY    = 1213;
156    protected static final int MULTIPLE_SPACES   = 1214;
157    // CHECKSTYLE.ON: SingleSpaceSeparator
158    // 1250 and up is used by tagcheck
159
160    protected EditableList sourcesList;
161
162    private static final List<String> DEFAULT_SOURCES = Arrays.asList(/*DATA_FILE, */IGNORE_FILE, SPELL_FILE);
163
164    /**
165     * Constructor
166     */
167    public TagChecker() {
168        super(tr("Tag checker"), tr("This test checks for errors in tag keys and values."));
169    }
170
171    @Override
172    public void initialize() throws IOException {
173        initializeData();
174        initializePresets();
175    }
176
177    /**
178     * Reads the spellcheck file into a HashMap.
179     * The data file is a list of words, beginning with +/-. If it starts with +,
180     * the word is valid, but if it starts with -, the word should be replaced
181     * by the nearest + word before this.
182     *
183     * @throws IOException if any I/O error occurs
184     */
185    private static void initializeData() throws IOException {
186        checkerData.clear();
187        ignoreDataStartsWith.clear();
188        ignoreDataEquals.clear();
189        ignoreDataEndsWith.clear();
190        ignoreDataTag.clear();
191        harmonizedKeys.clear();
192
193        StringBuilder errorSources = new StringBuilder();
194        for (String source : Config.getPref().getList(PREF_SOURCES, DEFAULT_SOURCES)) {
195            try (
196                CachedFile cf = new CachedFile(source);
197                BufferedReader reader = cf.getContentReader()
198            ) {
199                String okValue = null;
200                boolean tagcheckerfile = false;
201                boolean ignorefile = false;
202                boolean isFirstLine = true;
203                String line;
204                while ((line = reader.readLine()) != null && (tagcheckerfile || !line.isEmpty())) {
205                    if (line.startsWith("#")) {
206                        if (line.startsWith("# JOSM TagChecker")) {
207                            tagcheckerfile = true;
208                            if (!DEFAULT_SOURCES.contains(source)) {
209                                Logging.info(tr("Adding {0} to tag checker", source));
210                            }
211                        } else
212                        if (line.startsWith("# JOSM IgnoreTags")) {
213                            ignorefile = true;
214                            if (!DEFAULT_SOURCES.contains(source)) {
215                                Logging.info(tr("Adding {0} to ignore tags", source));
216                            }
217                        }
218                    } else if (ignorefile) {
219                        line = line.trim();
220                        if (line.length() < 4) {
221                            continue;
222                        }
223
224                        String key = line.substring(0, 2);
225                        line = line.substring(2);
226
227                        switch (key) {
228                        case "S:":
229                            ignoreDataStartsWith.add(line);
230                            break;
231                        case "E:":
232                            ignoreDataEquals.add(line);
233                            break;
234                        case "F:":
235                            ignoreDataEndsWith.add(line);
236                            break;
237                        case "K:":
238                            ignoreDataTag.add(Tag.ofString(line));
239                            break;
240                        default:
241                            if (!key.startsWith(";")) {
242                                Logging.warn("Unsupported TagChecker key: " + key);
243                            }
244                        }
245                    } else if (tagcheckerfile) {
246                        if (!line.isEmpty()) {
247                            CheckerData d = new CheckerData();
248                            String err = d.getData(line);
249
250                            if (err == null) {
251                                checkerData.add(d);
252                            } else {
253                                Logging.error(tr("Invalid tagchecker line - {0}: {1}", err, line));
254                            }
255                        }
256                    } else if (line.charAt(0) == '+') {
257                        okValue = line.substring(1);
258                    } else if (line.charAt(0) == '-' && okValue != null) {
259                        harmonizedKeys.put(harmonizeKey(line.substring(1)), okValue);
260                    } else {
261                        Logging.error(tr("Invalid spellcheck line: {0}", line));
262                    }
263                    if (isFirstLine) {
264                        isFirstLine = false;
265                        if (!(tagcheckerfile || ignorefile) && !DEFAULT_SOURCES.contains(source)) {
266                            Logging.info(tr("Adding {0} to spellchecker", source));
267                        }
268                    }
269                }
270            } catch (IOException e) {
271                Logging.error(e);
272                errorSources.append(source).append('\n');
273            }
274        }
275
276        if (errorSources.length() > 0)
277            throw new IOException(tr("Could not access data file(s):\n{0}", errorSources));
278    }
279
280    /**
281     * Reads the presets data.
282     *
283     */
284    public static void initializePresets() {
285
286        if (!Config.getPref().getBoolean(PREF_CHECK_VALUES, true))
287            return;
288
289        Collection<TaggingPreset> presets = TaggingPresets.getTaggingPresets();
290        if (!presets.isEmpty()) {
291            additionalPresetsValueData = new MultiMap<>();
292            for (String a : AbstractPrimitive.getUninterestingKeys()) {
293                additionalPresetsValueData.putVoid(a);
294            }
295            // TODO directionKeys are no longer in OsmPrimitive (search pattern is used instead)
296            for (String a : Config.getPref().getList(ValidatorPrefHelper.PREFIX + ".knownkeys",
297                    Arrays.asList("is_in", "int_ref", "fixme", "population"))) {
298                additionalPresetsValueData.putVoid(a);
299            }
300            for (TaggingPreset p : presets) {
301                for (TaggingPresetItem i : p.data) {
302                    if (i instanceof KeyedItem) {
303                        addPresetValue((KeyedItem) i);
304                    } else if (i instanceof CheckGroup) {
305                        for (Check c : ((CheckGroup) i).checks) {
306                            addPresetValue(c);
307                        }
308                    }
309                }
310            }
311        }
312    }
313
314    private static void addPresetValue(KeyedItem ky) {
315        Collection<String> values = ky.getValues();
316        if (ky.key != null && values != null) {
317            harmonizedKeys.put(harmonizeKey(ky.key), ky.key);
318        }
319    }
320
321    /**
322     * Checks given string (key or value) if it contains characters with code below 0x20 (either newline or some other special characters)
323     * @param s string to check
324     * @return {@code true} if {@code s} contains characters with code below 0x20
325     */
326    private static boolean containsLow(String s) {
327        if (s == null)
328            return false;
329        for (int i = 0; i < s.length(); i++) {
330            if (s.charAt(i) < 0x20)
331                return true;
332        }
333        return false;
334    }
335
336    private static Set<String> getPresetValues(String key) {
337        Set<String> res = TaggingPresets.getPresetValues(key);
338        if (res != null)
339            return res;
340        return additionalPresetsValueData.get(key);
341    }
342
343    /**
344     * Determines if the given key is in internal presets.
345     * @param key key
346     * @return {@code true} if the given key is in internal presets
347     * @since 9023
348     */
349    public static boolean isKeyInPresets(String key) {
350        return getPresetValues(key) != null;
351    }
352
353    /**
354     * Determines if the given tag is in internal presets.
355     * @param key key
356     * @param value value
357     * @return {@code true} if the given tag is in internal presets
358     * @since 9023
359     */
360    public static boolean isTagInPresets(String key, String value) {
361        final Set<String> values = getPresetValues(key);
362        return values != null && (values.isEmpty() || values.contains(value));
363    }
364
365    /**
366     * Returns the list of ignored tags.
367     * @return the list of ignored tags
368     * @since 9023
369     */
370    public static List<Tag> getIgnoredTags() {
371        return new ArrayList<>(ignoreDataTag);
372    }
373
374    /**
375     * Determines if the given tag is ignored for checks "key/tag not in presets".
376     * @param key key
377     * @param value value
378     * @return {@code true} if the given tag is ignored
379     * @since 9023
380     */
381    public static boolean isTagIgnored(String key, String value) {
382        boolean tagInPresets = isTagInPresets(key, value);
383        boolean ignore = false;
384
385        for (String a : ignoreDataStartsWith) {
386            if (key.startsWith(a)) {
387                ignore = true;
388            }
389        }
390        for (String a : ignoreDataEquals) {
391            if (key.equals(a)) {
392                ignore = true;
393            }
394        }
395        for (String a : ignoreDataEndsWith) {
396            if (key.endsWith(a)) {
397                ignore = true;
398            }
399        }
400
401        if (!tagInPresets) {
402            for (Tag a : ignoreDataTag) {
403                if (key.equals(a.getKey()) && value.equals(a.getValue())) {
404                    ignore = true;
405                }
406            }
407        }
408        return ignore;
409    }
410
411    /**
412     * Checks the primitive tags
413     * @param p The primitive to check
414     */
415    @Override
416    public void check(OsmPrimitive p) {
417        // Just a collection to know if a primitive has been already marked with error
418        MultiMap<OsmPrimitive, String> withErrors = new MultiMap<>();
419
420        if (checkComplex) {
421            Map<String, String> keys = p.getKeys();
422            for (CheckerData d : checkerData) {
423                if (d.match(p, keys)) {
424                    errors.add(TestError.builder(this, d.getSeverity(), d.getCode())
425                            .message(tr("Suspicious tag/value combinations"), d.getDescription())
426                            .primitives(p)
427                            .build());
428                    withErrors.put(p, "TC");
429                }
430            }
431        }
432
433        for (Entry<String, String> prop : p.getKeys().entrySet()) {
434            String s = marktr("Tag ''{0}'' invalid.");
435            String key = prop.getKey();
436            String value = prop.getValue();
437            if (checkValues && (containsLow(value)) && !withErrors.contains(p, "ICV")) {
438                errors.add(TestError.builder(this, Severity.WARNING, LOW_CHAR_VALUE)
439                        .message(tr("Tag value contains character with code less than 0x20"), s, key)
440                        .primitives(p)
441                        .build());
442                withErrors.put(p, "ICV");
443            }
444            if (checkKeys && (containsLow(key)) && !withErrors.contains(p, "ICK")) {
445                errors.add(TestError.builder(this, Severity.WARNING, LOW_CHAR_KEY)
446                        .message(tr("Tag key contains character with code less than 0x20"), s, key)
447                        .primitives(p)
448                        .build());
449                withErrors.put(p, "ICK");
450            }
451            if (checkValues && (value != null && value.length() > Tagged.MAX_TAG_LENGTH) && !withErrors.contains(p, "LV")) {
452                errors.add(TestError.builder(this, Severity.ERROR, LONG_VALUE)
453                        .message(tr("Tag value longer than {0} characters ({1} characters)", Tagged.MAX_TAG_LENGTH, value.length()), s, key)
454                        .primitives(p)
455                        .build());
456                withErrors.put(p, "LV");
457            }
458            if (checkKeys && (key != null && key.length() > Tagged.MAX_TAG_LENGTH) && !withErrors.contains(p, "LK")) {
459                errors.add(TestError.builder(this, Severity.ERROR, LONG_KEY)
460                        .message(tr("Tag key longer than {0} characters ({1} characters)", Tagged.MAX_TAG_LENGTH, key.length()), s, key)
461                        .primitives(p)
462                        .build());
463                withErrors.put(p, "LK");
464            }
465            if (checkValues && (value == null || value.trim().isEmpty()) && !withErrors.contains(p, "EV")) {
466                errors.add(TestError.builder(this, Severity.WARNING, EMPTY_VALUES)
467                        .message(tr("Tags with empty values"), s, key)
468                        .primitives(p)
469                        .build());
470                withErrors.put(p, "EV");
471            }
472            if (checkKeys && key != null && key.indexOf(' ') >= 0 && !withErrors.contains(p, "IPK")) {
473                errors.add(TestError.builder(this, Severity.WARNING, INVALID_KEY_SPACE)
474                        .message(tr("Invalid white space in property key"), s, key)
475                        .primitives(p)
476                        .build());
477                withErrors.put(p, "IPK");
478            }
479            if (checkValues && value != null && (value.startsWith(" ") || value.endsWith(" ")) && !withErrors.contains(p, "SPACE")) {
480                errors.add(TestError.builder(this, Severity.WARNING, INVALID_SPACE)
481                        .message(tr("Property values start or end with white space"), s, key)
482                        .primitives(p)
483                        .build());
484                withErrors.put(p, "SPACE");
485            }
486            if (checkValues && value != null && value.contains("  ") && !withErrors.contains(p, "SPACE")) {
487                errors.add(TestError.builder(this, Severity.WARNING, MULTIPLE_SPACES)
488                        .message(tr("Property values contain multiple white spaces"), s, key)
489                        .primitives(p)
490                        .build());
491                withErrors.put(p, "SPACE");
492            }
493            if (checkValues && value != null && !value.equals(Entities.unescape(value)) && !withErrors.contains(p, "HTML")) {
494                errors.add(TestError.builder(this, Severity.OTHER, INVALID_HTML)
495                        .message(tr("Property values contain HTML entity"), s, key)
496                        .primitives(p)
497                        .build());
498                withErrors.put(p, "HTML");
499            }
500            if (checkValues && key != null && value != null && !value.isEmpty() && additionalPresetsValueData != null
501                    && !isTagIgnored(key, value)) {
502                if (!isKeyInPresets(key)) {
503                    String prettifiedKey = harmonizeKey(key);
504                    String fixedKey = harmonizedKeys.get(prettifiedKey);
505                    if (fixedKey != null && !"".equals(fixedKey) && !fixedKey.equals(key)) {
506                        // misspelled preset key
507                        final TestError.Builder error = TestError.builder(this, Severity.WARNING, MISSPELLED_KEY)
508                                .message(tr("Misspelled property key"), marktr("Key ''{0}'' looks like ''{1}''."), key, fixedKey)
509                                .primitives(p);
510                        if (p.hasKey(fixedKey)) {
511                            errors.add(error.build());
512                        } else {
513                            errors.add(error.fix(() -> new ChangePropertyKeyCommand(p, key, fixedKey)).build());
514                        }
515                        withErrors.put(p, "WPK");
516                    } else {
517                        errors.add(TestError.builder(this, Severity.OTHER, INVALID_VALUE)
518                                .message(tr("Presets do not contain property key"), marktr("Key ''{0}'' not in presets."), key)
519                                .primitives(p)
520                                .build());
521                        withErrors.put(p, "UPK");
522                    }
523                } else if (!isTagInPresets(key, value)) {
524                    // try to fix common typos and check again if value is still unknown
525                    String fixedValue = harmonizeValue(prop.getValue());
526                    Map<String, String> possibleValues = getPossibleValues(getPresetValues(key));
527                    if (possibleValues.containsKey(fixedValue)) {
528                        final String newValue = possibleValues.get(fixedValue);
529                        // misspelled preset value
530                        errors.add(TestError.builder(this, Severity.WARNING, MISSPELLED_VALUE)
531                                .message(tr("Misspelled property value"),
532                                        marktr("Value ''{0}'' for key ''{1}'' looks like ''{2}''."), prop.getValue(), key, newValue)
533                                .primitives(p)
534                                .fix(() -> new ChangePropertyCommand(p, key, newValue))
535                                .build());
536                        withErrors.put(p, "WPV");
537                    } else {
538                        // unknown preset value
539                        errors.add(TestError.builder(this, Severity.OTHER, INVALID_VALUE)
540                                .message(tr("Presets do not contain property value"),
541                                        marktr("Value ''{0}'' for key ''{1}'' not in presets."), prop.getValue(), key)
542                                .primitives(p)
543                                .build());
544                        withErrors.put(p, "UPV");
545                    }
546                }
547            }
548            if (checkFixmes && key != null && value != null && !value.isEmpty() && isFixme(key, value) && !withErrors.contains(p, "FIXME")) {
549               errors.add(TestError.builder(this, Severity.OTHER, FIXME)
550                .message(tr("FIXMES"))
551                .primitives(p)
552                .build());
553               withErrors.put(p, "FIXME");
554            }
555        }
556    }
557
558    private static boolean isFixme(String key, String value) {
559        return key.toLowerCase(Locale.ENGLISH).contains("fixme") || key.contains("todo")
560          || value.toLowerCase(Locale.ENGLISH).contains("fixme") || value.contains("check and delete");
561    }
562
563    private static Map<String, String> getPossibleValues(Set<String> values) {
564        // generate a map with common typos
565        Map<String, String> map = new HashMap<>();
566        if (values != null) {
567            for (String value : values) {
568                map.put(value, value);
569                if (value.contains("_")) {
570                    map.put(value.replace("_", ""), value);
571                }
572            }
573        }
574        return map;
575    }
576
577    private static String harmonizeKey(String key) {
578        return Utils.strip(key.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(':', '_').replace(' ', '_'), "-_;:,");
579    }
580
581    private static String harmonizeValue(String value) {
582        return Utils.strip(value.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(' ', '_'), "-_;:,");
583    }
584
585    @Override
586    public void startTest(ProgressMonitor monitor) {
587        super.startTest(monitor);
588        checkKeys = Config.getPref().getBoolean(PREF_CHECK_KEYS, true);
589        if (isBeforeUpload) {
590            checkKeys = checkKeys && Config.getPref().getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true);
591        }
592
593        checkValues = Config.getPref().getBoolean(PREF_CHECK_VALUES, true);
594        if (isBeforeUpload) {
595            checkValues = checkValues && Config.getPref().getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true);
596        }
597
598        checkComplex = Config.getPref().getBoolean(PREF_CHECK_COMPLEX, true);
599        if (isBeforeUpload) {
600            checkComplex = checkComplex && Config.getPref().getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true);
601        }
602
603        checkFixmes = Config.getPref().getBoolean(PREF_CHECK_FIXMES, true);
604        if (isBeforeUpload) {
605            checkFixmes = checkFixmes && Config.getPref().getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true);
606        }
607    }
608
609    @Override
610    public void visit(Collection<OsmPrimitive> selection) {
611        if (checkKeys || checkValues || checkComplex || checkFixmes) {
612            super.visit(selection);
613        }
614    }
615
616    @Override
617    public void addGui(JPanel testPanel) {
618        GBC a = GBC.eol();
619        a.anchor = GridBagConstraints.EAST;
620
621        testPanel.add(new JLabel(name+" :"), GBC.eol().insets(3, 0, 0, 0));
622
623        prefCheckKeys = new JCheckBox(tr("Check property keys."), Config.getPref().getBoolean(PREF_CHECK_KEYS, true));
624        prefCheckKeys.setToolTipText(tr("Validate that property keys are valid checking against list of words."));
625        testPanel.add(prefCheckKeys, GBC.std().insets(20, 0, 0, 0));
626
627        prefCheckKeysBeforeUpload = new JCheckBox();
628        prefCheckKeysBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true));
629        testPanel.add(prefCheckKeysBeforeUpload, a);
630
631        prefCheckComplex = new JCheckBox(tr("Use complex property checker."), Config.getPref().getBoolean(PREF_CHECK_COMPLEX, true));
632        prefCheckComplex.setToolTipText(tr("Validate property values and tags using complex rules."));
633        testPanel.add(prefCheckComplex, GBC.std().insets(20, 0, 0, 0));
634
635        prefCheckComplexBeforeUpload = new JCheckBox();
636        prefCheckComplexBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true));
637        testPanel.add(prefCheckComplexBeforeUpload, a);
638
639        final Collection<String> sources = Config.getPref().getList(PREF_SOURCES, DEFAULT_SOURCES);
640        sourcesList = new EditableList(tr("TagChecker source"));
641        sourcesList.setItems(sources);
642        testPanel.add(new JLabel(tr("Data sources ({0})", "*.cfg")), GBC.eol().insets(23, 0, 0, 0));
643        testPanel.add(sourcesList, GBC.eol().fill(GridBagConstraints.HORIZONTAL).insets(23, 0, 0, 0));
644
645        ActionListener disableCheckActionListener = e -> handlePrefEnable();
646        prefCheckKeys.addActionListener(disableCheckActionListener);
647        prefCheckKeysBeforeUpload.addActionListener(disableCheckActionListener);
648        prefCheckComplex.addActionListener(disableCheckActionListener);
649        prefCheckComplexBeforeUpload.addActionListener(disableCheckActionListener);
650
651        handlePrefEnable();
652
653        prefCheckValues = new JCheckBox(tr("Check property values."), Config.getPref().getBoolean(PREF_CHECK_VALUES, true));
654        prefCheckValues.setToolTipText(tr("Validate that property values are valid checking against presets."));
655        testPanel.add(prefCheckValues, GBC.std().insets(20, 0, 0, 0));
656
657        prefCheckValuesBeforeUpload = new JCheckBox();
658        prefCheckValuesBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true));
659        testPanel.add(prefCheckValuesBeforeUpload, a);
660
661        prefCheckFixmes = new JCheckBox(tr("Check for FIXMES."), Config.getPref().getBoolean(PREF_CHECK_FIXMES, true));
662        prefCheckFixmes.setToolTipText(tr("Looks for nodes or ways with FIXME in any property value."));
663        testPanel.add(prefCheckFixmes, GBC.std().insets(20, 0, 0, 0));
664
665        prefCheckFixmesBeforeUpload = new JCheckBox();
666        prefCheckFixmesBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true));
667        testPanel.add(prefCheckFixmesBeforeUpload, a);
668    }
669
670    /**
671     * Enables/disables the source list field
672     */
673    public void handlePrefEnable() {
674        boolean selected = prefCheckKeys.isSelected() || prefCheckKeysBeforeUpload.isSelected()
675                || prefCheckComplex.isSelected() || prefCheckComplexBeforeUpload.isSelected();
676        sourcesList.setEnabled(selected);
677    }
678
679    @Override
680    public boolean ok() {
681        enabled = prefCheckKeys.isSelected() || prefCheckValues.isSelected() || prefCheckComplex.isSelected() || prefCheckFixmes.isSelected();
682        testBeforeUpload = prefCheckKeysBeforeUpload.isSelected() || prefCheckValuesBeforeUpload.isSelected()
683                || prefCheckFixmesBeforeUpload.isSelected() || prefCheckComplexBeforeUpload.isSelected();
684
685        Config.getPref().putBoolean(PREF_CHECK_VALUES, prefCheckValues.isSelected());
686        Config.getPref().putBoolean(PREF_CHECK_COMPLEX, prefCheckComplex.isSelected());
687        Config.getPref().putBoolean(PREF_CHECK_KEYS, prefCheckKeys.isSelected());
688        Config.getPref().putBoolean(PREF_CHECK_FIXMES, prefCheckFixmes.isSelected());
689        Config.getPref().putBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, prefCheckValuesBeforeUpload.isSelected());
690        Config.getPref().putBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, prefCheckComplexBeforeUpload.isSelected());
691        Config.getPref().putBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, prefCheckKeysBeforeUpload.isSelected());
692        Config.getPref().putBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, prefCheckFixmesBeforeUpload.isSelected());
693        return Config.getPref().putList(PREF_SOURCES, sourcesList.getItems());
694    }
695
696    @Override
697    public Command fixError(TestError testError) {
698        List<Command> commands = new ArrayList<>(50);
699
700        Collection<? extends OsmPrimitive> primitives = testError.getPrimitives();
701        for (OsmPrimitive p : primitives) {
702            Map<String, String> tags = p.getKeys();
703            if (tags.isEmpty()) {
704                continue;
705            }
706
707            for (Entry<String, String> prop: tags.entrySet()) {
708                String key = prop.getKey();
709                String value = prop.getValue();
710                if (value == null || value.trim().isEmpty()) {
711                    commands.add(new ChangePropertyCommand(p, key, null));
712                } else if (value.startsWith(" ") || value.endsWith(" ") || value.contains("  ")) {
713                    commands.add(new ChangePropertyCommand(p, key, Utils.removeWhiteSpaces(value)));
714                } else if (key.startsWith(" ") || key.endsWith(" ") || key.contains("  ")) {
715                    commands.add(new ChangePropertyKeyCommand(p, key, Utils.removeWhiteSpaces(key)));
716                } else {
717                    String evalue = Entities.unescape(value);
718                    if (!evalue.equals(value)) {
719                        commands.add(new ChangePropertyCommand(p, key, evalue));
720                    }
721                }
722            }
723        }
724
725        if (commands.isEmpty())
726            return null;
727        if (commands.size() == 1)
728            return commands.get(0);
729
730        return new SequenceCommand(tr("Fix tags"), commands);
731    }
732
733    @Override
734    public boolean isFixable(TestError testError) {
735        if (testError.getTester() instanceof TagChecker) {
736            int code = testError.getCode();
737            return code == INVALID_KEY || code == EMPTY_VALUES || code == INVALID_SPACE ||
738                   code == INVALID_KEY_SPACE || code == INVALID_HTML || code == MISSPELLED_VALUE ||
739                   code == MULTIPLE_SPACES;
740        }
741
742        return false;
743    }
744
745    protected static class CheckerData {
746        private String description;
747        protected List<CheckerElement> data = new ArrayList<>();
748        private OsmPrimitiveType type;
749        private TagCheckLevel level;
750        protected Severity severity;
751
752        private enum TagCheckLevel {
753            TAG_CHECK_ERROR(1250),
754            TAG_CHECK_WARN(1260),
755            TAG_CHECK_INFO(1270);
756
757            final int code;
758
759            TagCheckLevel(int code) {
760                this.code = code;
761            }
762        }
763
764        protected static class CheckerElement {
765            public Object tag;
766            public Object value;
767            public boolean noMatch;
768            public boolean tagAll;
769            public boolean valueAll;
770            public boolean valueBool;
771
772            private static Pattern getPattern(String str) {
773                if (str.endsWith("/i"))
774                    return Pattern.compile(str.substring(1, str.length()-2), Pattern.CASE_INSENSITIVE);
775                if (str.endsWith("/"))
776                    return Pattern.compile(str.substring(1, str.length()-1));
777
778                throw new IllegalStateException();
779            }
780
781            public CheckerElement(String exp) {
782                Matcher m = Pattern.compile("(.+)([!=]=)(.+)").matcher(exp);
783                m.matches();
784
785                String n = m.group(1).trim();
786
787                if ("*".equals(n)) {
788                    tagAll = true;
789                } else {
790                    tag = n.startsWith("/") ? getPattern(n) : n;
791                    noMatch = "!=".equals(m.group(2));
792                    n = m.group(3).trim();
793                    if ("*".equals(n)) {
794                        valueAll = true;
795                    } else if ("BOOLEAN_TRUE".equals(n)) {
796                        valueBool = true;
797                        value = OsmUtils.TRUE_VALUE;
798                    } else if ("BOOLEAN_FALSE".equals(n)) {
799                        valueBool = true;
800                        value = OsmUtils.FALSE_VALUE;
801                    } else {
802                        value = n.startsWith("/") ? getPattern(n) : n;
803                    }
804                }
805            }
806
807            public boolean match(Map<String, String> keys) {
808                for (Entry<String, String> prop: keys.entrySet()) {
809                    String key = prop.getKey();
810                    String val = valueBool ? OsmUtils.getNamedOsmBoolean(prop.getValue()) : prop.getValue();
811                    if ((tagAll || (tag instanceof Pattern ? ((Pattern) tag).matcher(key).matches() : key.equals(tag)))
812                            && (valueAll || (value instanceof Pattern ? ((Pattern) value).matcher(val).matches() : val.equals(value))))
813                        return !noMatch;
814                }
815                return noMatch;
816            }
817        }
818
819        private static final Pattern CLEAN_STR_PATTERN = Pattern.compile(" *# *([^#]+) *$");
820        private static final Pattern SPLIT_TRIMMED_PATTERN = Pattern.compile(" *: *");
821        private static final Pattern SPLIT_ELEMENTS_PATTERN = Pattern.compile(" *&& *");
822
823        public String getData(final String str) {
824            Matcher m = CLEAN_STR_PATTERN.matcher(str);
825            String trimmed = m.replaceFirst("").trim();
826            try {
827                description = m.group(1);
828                if (description != null && description.isEmpty()) {
829                    description = null;
830                }
831            } catch (IllegalStateException e) {
832                Logging.error(e);
833                description = null;
834            }
835            String[] n = SPLIT_TRIMMED_PATTERN.split(trimmed, 3);
836            switch (n[0]) {
837            case "way":
838                type = OsmPrimitiveType.WAY;
839                break;
840            case "node":
841                type = OsmPrimitiveType.NODE;
842                break;
843            case "relation":
844                type = OsmPrimitiveType.RELATION;
845                break;
846            case "*":
847                type = null;
848                break;
849            default:
850                return tr("Could not find element type");
851            }
852            if (n.length != 3)
853                return tr("Incorrect number of parameters");
854
855            switch (n[1]) {
856            case "W":
857                severity = Severity.WARNING;
858                level = TagCheckLevel.TAG_CHECK_WARN;
859                break;
860            case "E":
861                severity = Severity.ERROR;
862                level = TagCheckLevel.TAG_CHECK_ERROR;
863                break;
864            case "I":
865                severity = Severity.OTHER;
866                level = TagCheckLevel.TAG_CHECK_INFO;
867                break;
868            default:
869                return tr("Could not find warning level");
870            }
871            for (String exp: SPLIT_ELEMENTS_PATTERN.split(n[2])) {
872                try {
873                    data.add(new CheckerElement(exp));
874                } catch (IllegalStateException e) {
875                    Logging.trace(e);
876                    return tr("Illegal expression ''{0}''", exp);
877                } catch (PatternSyntaxException e) {
878                    Logging.trace(e);
879                    return tr("Illegal regular expression ''{0}''", exp);
880                }
881            }
882            return null;
883        }
884
885        public boolean match(OsmPrimitive osm, Map<String, String> keys) {
886            if (type != null && OsmPrimitiveType.from(osm) != type)
887                return false;
888
889            for (CheckerElement ce : data) {
890                if (!ce.match(keys))
891                    return false;
892            }
893            return true;
894        }
895
896        /**
897         * Returns the error description.
898         * @return the error description
899         */
900        public String getDescription() {
901            return description;
902        }
903
904        /**
905         * Returns the error severity.
906         * @return the error severity
907         */
908        public Severity getSeverity() {
909            return severity;
910        }
911
912        /**
913         * Returns the error code.
914         * @return the error code
915         */
916        public int getCode() {
917            if (type == null)
918                return level.code;
919
920            return level.code + type.ordinal() + 1;
921        }
922    }
923}