001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.gui.layer;
003
004import static org.openstreetmap.josm.tools.I18n.marktr;
005import static org.openstreetmap.josm.tools.I18n.tr;
006import static org.openstreetmap.josm.tools.Utils.getSystemProperty;
007
008import java.io.BufferedReader;
009import java.io.File;
010import java.io.FileFilter;
011import java.io.IOException;
012import java.io.PrintStream;
013import java.lang.management.ManagementFactory;
014import java.nio.charset.StandardCharsets;
015import java.nio.file.Files;
016import java.nio.file.Path;
017import java.util.ArrayList;
018import java.util.Date;
019import java.util.Deque;
020import java.util.HashSet;
021import java.util.Iterator;
022import java.util.LinkedList;
023import java.util.List;
024import java.util.Locale;
025import java.util.Set;
026import java.util.Timer;
027import java.util.TimerTask;
028import java.util.concurrent.ExecutionException;
029import java.util.concurrent.Future;
030import java.util.concurrent.TimeUnit;
031import java.util.regex.Pattern;
032
033import org.openstreetmap.josm.actions.OpenFileAction.OpenFileTask;
034import org.openstreetmap.josm.data.osm.DataSet;
035import org.openstreetmap.josm.data.osm.NoteData;
036import org.openstreetmap.josm.data.osm.NoteData.NoteDataUpdateListener;
037import org.openstreetmap.josm.data.osm.event.AbstractDatasetChangedEvent;
038import org.openstreetmap.josm.data.osm.event.DataSetListenerAdapter;
039import org.openstreetmap.josm.data.osm.event.DataSetListenerAdapter.Listener;
040import org.openstreetmap.josm.data.preferences.BooleanProperty;
041import org.openstreetmap.josm.data.preferences.IntegerProperty;
042import org.openstreetmap.josm.gui.MainApplication;
043import org.openstreetmap.josm.gui.Notification;
044import org.openstreetmap.josm.gui.io.importexport.NoteExporter;
045import org.openstreetmap.josm.gui.io.importexport.NoteImporter;
046import org.openstreetmap.josm.gui.io.importexport.OsmExporter;
047import org.openstreetmap.josm.gui.io.importexport.OsmImporter;
048import org.openstreetmap.josm.gui.layer.LayerManager.LayerAddEvent;
049import org.openstreetmap.josm.gui.layer.LayerManager.LayerChangeListener;
050import org.openstreetmap.josm.gui.layer.LayerManager.LayerOrderChangeEvent;
051import org.openstreetmap.josm.gui.layer.LayerManager.LayerRemoveEvent;
052import org.openstreetmap.josm.gui.util.GuiHelper;
053import org.openstreetmap.josm.spi.preferences.Config;
054import org.openstreetmap.josm.tools.Logging;
055import org.openstreetmap.josm.tools.Utils;
056
057/**
058 * Saves data and note layers periodically so they can be recovered in case of a crash.
059 *
060 * There are 2 directories
061 *  - autosave dir: copies of the currently open data layers are saved here every
062 *      PROP_INTERVAL seconds. When a data layer is closed normally, the corresponding
063 *      files are removed. If this dir is non-empty on start, JOSM assumes
064 *      that it crashed last time.
065 *  - deleted layers dir: "secondary archive" - when autosaved layers are restored
066 *      they are copied to this directory. We cannot keep them in the autosave folder,
067 *      but just deleting it would be dangerous: Maybe a feature inside the file
068 *      caused JOSM to crash. If the data is valuable, the user can still try to
069 *      open with another versions of JOSM or fix the problem manually.
070 *
071 *      The deleted layers dir keeps at most PROP_DELETED_LAYERS files.
072 *
073 * @since  3378 (creation)
074 * @since 10386 (new LayerChangeListener interface)
075 */
076public class AutosaveTask extends TimerTask implements LayerChangeListener, Listener, NoteDataUpdateListener {
077
078    private static final char[] ILLEGAL_CHARACTERS = {'/', '\n', '\r', '\t', '\0', '\f', '`', '?', '*', '\\', '<', '>', '|', '\"', ':'};
079    private static final String AUTOSAVE_DIR = "autosave";
080    private static final String DELETED_LAYERS_DIR = "autosave/deleted_layers";
081
082    /**
083     * If autosave is enabled
084     */
085    public static final BooleanProperty PROP_AUTOSAVE_ENABLED = new BooleanProperty("autosave.enabled", true);
086    /**
087     * The number of files to store per layer
088     */
089    public static final IntegerProperty PROP_FILES_PER_LAYER = new IntegerProperty("autosave.filesPerLayer", 1);
090    /**
091     * How many deleted layers should be stored
092     */
093    public static final IntegerProperty PROP_DELETED_LAYERS = new IntegerProperty("autosave.deletedLayersBackupCount", 5);
094    /**
095     * The autosave interval, in seconds
096     */
097    public static final IntegerProperty PROP_INTERVAL = new IntegerProperty("autosave.interval", (int) TimeUnit.MINUTES.toSeconds(5));
098    /**
099     * The maximum number of autosave files to store
100     */
101    public static final IntegerProperty PROP_INDEX_LIMIT = new IntegerProperty("autosave.index-limit", 1000);
102    /**
103     * Defines if a notification should be displayed after each autosave
104     */
105    public static final BooleanProperty PROP_NOTIFICATION = new BooleanProperty("autosave.notification", false);
106
107    protected static final class AutosaveLayerInfo<T extends AbstractModifiableLayer> {
108        private final T layer;
109        private String layerName;
110        private String layerFileName;
111        private final Deque<File> backupFiles = new LinkedList<>();
112
113        AutosaveLayerInfo(T layer) {
114            this.layer = layer;
115        }
116    }
117
118    private final DataSetListenerAdapter datasetAdapter = new DataSetListenerAdapter(this);
119    private final Set<DataSet> changedDatasets = new HashSet<>();
120    private final Set<NoteData> changedNoteData = new HashSet<>();
121    private final List<AutosaveLayerInfo<?>> layersInfo = new ArrayList<>();
122    private final Object layersLock = new Object();
123    private final Deque<File> deletedLayers = new LinkedList<>();
124
125    private final File autosaveDir = new File(Config.getDirs().getUserDataDirectory(true), AUTOSAVE_DIR);
126    private final File deletedLayersDir = new File(Config.getDirs().getUserDataDirectory(true), DELETED_LAYERS_DIR);
127
128    /**
129     * Replies the autosave directory.
130     * @return the autosave directory
131     * @since 10299
132     */
133    public final Path getAutosaveDir() {
134        return autosaveDir.toPath();
135    }
136
137    /**
138     * Starts the autosave background task.
139     */
140    public void schedule() {
141        if (PROP_INTERVAL.get() > 0) {
142
143            if (!autosaveDir.exists() && !autosaveDir.mkdirs()) {
144                Logging.warn(tr("Unable to create directory {0}, autosave will be disabled", autosaveDir.getAbsolutePath()));
145                return;
146            }
147            if (!deletedLayersDir.exists() && !deletedLayersDir.mkdirs()) {
148                Logging.warn(tr("Unable to create directory {0}, autosave will be disabled", deletedLayersDir.getAbsolutePath()));
149                return;
150            }
151
152            File[] files = deletedLayersDir.listFiles();
153            if (files != null) {
154                for (File f: files) {
155                    deletedLayers.add(f); // FIXME: sort by mtime
156                }
157            }
158
159            new Timer(true).schedule(this, TimeUnit.SECONDS.toMillis(1), TimeUnit.SECONDS.toMillis(PROP_INTERVAL.get()));
160            MainApplication.getLayerManager().addAndFireLayerChangeListener(this);
161        }
162    }
163
164    private static String getFileName(String layerName, int index) {
165        String result = layerName;
166        for (char illegalCharacter : ILLEGAL_CHARACTERS) {
167            result = result.replaceAll(Pattern.quote(String.valueOf(illegalCharacter)),
168                    '&' + String.valueOf((int) illegalCharacter) + ';');
169        }
170        if (index != 0) {
171            result = result + '_' + index;
172        }
173        return result;
174    }
175
176    private void setLayerFileName(AutosaveLayerInfo<?> layer) {
177        int index = 0;
178        while (true) {
179            String filename = getFileName(layer.layer.getName(), index);
180            boolean foundTheSame = false;
181            for (AutosaveLayerInfo<?> info: layersInfo) {
182                if (info != layer && filename.equals(info.layerFileName)) {
183                    foundTheSame = true;
184                    break;
185                }
186            }
187
188            if (!foundTheSame) {
189                layer.layerFileName = filename;
190                return;
191            }
192
193            index++;
194        }
195    }
196
197    protected File getNewLayerFile(AutosaveLayerInfo<?> layer, Date now, int startIndex) {
198        int index = startIndex;
199        while (true) {
200            String filename = String.format(Locale.ENGLISH, "%1$s_%2$tY%2$tm%2$td_%2$tH%2$tM%2$tS%2$tL%3$s",
201                    layer.layerFileName, now, index == 0 ? "" : ('_' + Integer.toString(index)));
202            File result = new File(autosaveDir, filename + '.' +
203                    (layer.layer instanceof NoteLayer ?
204                            Config.getPref().get("autosave.notes.extension", "osn") :
205                            Config.getPref().get("autosave.extension", "osm")));
206            try {
207                if (index > PROP_INDEX_LIMIT.get())
208                    throw new IOException("index limit exceeded");
209                if (result.createNewFile()) {
210                    createNewPidFile(autosaveDir, filename);
211                    return result;
212                } else {
213                    Logging.warn(tr("Unable to create file {0}, other filename will be used", result.getAbsolutePath()));
214                }
215            } catch (IOException e) {
216                Logging.log(Logging.LEVEL_ERROR, tr("IOError while creating file, autosave will be skipped: {0}", e.getMessage()), e);
217                return null;
218            }
219            index++;
220        }
221    }
222
223    private static void createNewPidFile(File autosaveDir, String filename) {
224        File pidFile = new File(autosaveDir, filename+".pid");
225        try (PrintStream ps = new PrintStream(pidFile, "UTF-8")) {
226            ps.println(ManagementFactory.getRuntimeMXBean().getName());
227        } catch (IOException | SecurityException t) {
228            Logging.error(t);
229        }
230    }
231
232    private void savelayer(AutosaveLayerInfo<?> info) {
233        if (!info.layer.getName().equals(info.layerName)) {
234            setLayerFileName(info);
235            info.layerName = info.layer.getName();
236        }
237        try {
238            if (info.layer instanceof OsmDataLayer) {
239                OsmDataLayer dataLayer = (OsmDataLayer) info.layer;
240                if (changedDatasets.remove(dataLayer.data)) {
241                    File file = getNewLayerFile(info, new Date(), 0);
242                    if (file != null) {
243                        info.backupFiles.add(file);
244                        new OsmExporter().exportData(file, info.layer, true /* no backup with appended ~ */);
245                    }
246                }
247            } else if (info.layer instanceof NoteLayer) {
248                NoteLayer noteLayer = (NoteLayer) info.layer;
249                if (changedNoteData.remove(noteLayer.getNoteData())) {
250                    File file = getNewLayerFile(info, new Date(), 0);
251                    if (file != null) {
252                        info.backupFiles.add(file);
253                        new NoteExporter().exportData(file, info.layer);
254                    }
255                }
256            }
257        } catch (IOException e) {
258            Logging.error(e);
259        }
260        while (info.backupFiles.size() > PROP_FILES_PER_LAYER.get()) {
261            File oldFile = info.backupFiles.remove();
262            if (Utils.deleteFile(oldFile, marktr("Unable to delete old backup file {0}"))) {
263                Utils.deleteFile(getPidFile(oldFile), marktr("Unable to delete old backup file {0}"));
264            }
265        }
266    }
267
268    @Override
269    public void run() {
270        synchronized (layersLock) {
271            try {
272                for (AutosaveLayerInfo<?> info: layersInfo) {
273                    savelayer(info);
274                }
275                changedDatasets.clear();
276                changedNoteData.clear();
277                if (PROP_NOTIFICATION.get() && !layersInfo.isEmpty()) {
278                    GuiHelper.runInEDT(this::displayNotification);
279                }
280            } catch (RuntimeException t) { // NOPMD
281                // Don't let exception stop time thread
282                Logging.error("Autosave failed:");
283                Logging.error(t);
284            }
285        }
286    }
287
288    protected void displayNotification() {
289        new Notification(tr("Your work has been saved automatically."))
290        .setDuration(Notification.TIME_SHORT)
291        .show();
292    }
293
294    @Override
295    public void layerOrderChanged(LayerOrderChangeEvent e) {
296        // Do nothing
297    }
298
299    private void registerNewlayer(OsmDataLayer layer) {
300        synchronized (layersLock) {
301            layer.getDataSet().addDataSetListener(datasetAdapter);
302            layersInfo.add(new AutosaveLayerInfo<>(layer));
303        }
304    }
305
306    private void registerNewlayer(NoteLayer layer) {
307        synchronized (layersLock) {
308            layer.getNoteData().addNoteDataUpdateListener(this);
309            layersInfo.add(new AutosaveLayerInfo<>(layer));
310        }
311    }
312
313    @Override
314    public void layerAdded(LayerAddEvent e) {
315        if (e.getAddedLayer() instanceof OsmDataLayer) {
316            registerNewlayer((OsmDataLayer) e.getAddedLayer());
317        } else if (e.getAddedLayer() instanceof NoteLayer) {
318            registerNewlayer((NoteLayer) e.getAddedLayer());
319        }
320    }
321
322    @Override
323    public void layerRemoving(LayerRemoveEvent e) {
324        if (e.getRemovedLayer() instanceof OsmDataLayer) {
325            synchronized (layersLock) {
326                OsmDataLayer osmLayer = (OsmDataLayer) e.getRemovedLayer();
327                osmLayer.getDataSet().removeDataSetListener(datasetAdapter);
328                cleanupLayer(osmLayer);
329            }
330        } else if (e.getRemovedLayer() instanceof NoteLayer) {
331            synchronized (layersLock) {
332                NoteLayer noteLayer = (NoteLayer) e.getRemovedLayer();
333                noteLayer.getNoteData().removeNoteDataUpdateListener(this);
334                cleanupLayer(noteLayer);
335            }
336        }
337    }
338
339    private void cleanupLayer(AbstractModifiableLayer removedLayer) {
340        Iterator<AutosaveLayerInfo<?>> it = layersInfo.iterator();
341        while (it.hasNext()) {
342            AutosaveLayerInfo<?> info = it.next();
343            if (info.layer == removedLayer) {
344
345                savelayer(info);
346                File lastFile = info.backupFiles.pollLast();
347                if (lastFile != null) {
348                    moveToDeletedLayersFolder(lastFile);
349                }
350                for (File file: info.backupFiles) {
351                    if (Utils.deleteFile(file)) {
352                        Utils.deleteFile(getPidFile(file));
353                    }
354                }
355
356                it.remove();
357            }
358        }
359    }
360
361    @Override
362    public void processDatasetEvent(AbstractDatasetChangedEvent event) {
363        changedDatasets.add(event.getDataset());
364    }
365
366    @Override
367    public void noteDataUpdated(NoteData data) {
368        changedNoteData.add(data);
369    }
370
371    @Override
372    public void selectedNoteChanged(NoteData noteData) {
373        // Do nothing
374    }
375
376    protected File getPidFile(File osmFile) {
377        return new File(autosaveDir, osmFile.getName().replaceFirst("[.][^.]+$", ".pid"));
378    }
379
380    /**
381     * Replies the list of .osm files still present in autosave dir, that are not currently managed by another instance of JOSM.
382     * These files are hence unsaved layers from an old instance of JOSM that crashed and may be recovered by this instance.
383     * @return The list of .osm files still present in autosave dir, that are not currently managed by another instance of JOSM
384     */
385    public List<File> getUnsavedLayersFiles() {
386        List<File> result = new ArrayList<>();
387        try {
388            File[] files = autosaveDir.listFiles((FileFilter)
389                    pathname -> OsmImporter.FILE_FILTER.accept(pathname) || NoteImporter.FILE_FILTER.accept(pathname));
390            if (files == null)
391                return result;
392            for (File file: files) {
393                if (file.isFile()) {
394                    boolean skipFile = false;
395                    File pidFile = getPidFile(file);
396                    if (pidFile.exists()) {
397                        try (BufferedReader reader = Files.newBufferedReader(pidFile.toPath(), StandardCharsets.UTF_8)) {
398                            String jvmId = reader.readLine();
399                            if (jvmId != null) {
400                                String pid = jvmId.split("@")[0];
401                                skipFile = jvmPerfDataFileExists(pid);
402                            }
403                        } catch (IOException | SecurityException t) {
404                            Logging.error(t);
405                        }
406                    }
407                    if (!skipFile) {
408                        result.add(file);
409                    }
410                }
411            }
412        } catch (SecurityException e) {
413            Logging.log(Logging.LEVEL_ERROR, "Unable to list unsaved layers files", e);
414        }
415        return result;
416    }
417
418    private static boolean jvmPerfDataFileExists(final String jvmId) {
419        File jvmDir = new File(getSystemProperty("java.io.tmpdir") + File.separator + "hsperfdata_" + getSystemProperty("user.name"));
420        if (jvmDir.exists() && jvmDir.canRead()) {
421            File[] files = jvmDir.listFiles((FileFilter) file -> file.getName().equals(jvmId) && file.isFile());
422            return files != null && files.length == 1;
423        }
424        return false;
425    }
426
427    /**
428     * Recover the unsaved layers and open them asynchronously.
429     * @return A future that can be used to wait for the completion of this task.
430     */
431    public Future<?> recoverUnsavedLayers() {
432        List<File> files = getUnsavedLayersFiles();
433        final OpenFileTask openFileTsk = new OpenFileTask(files, null, tr("Restoring files"));
434        final Future<?> openFilesFuture = MainApplication.worker.submit(openFileTsk);
435        return MainApplication.worker.submit(() -> {
436            try {
437                // Wait for opened tasks to be generated.
438                openFilesFuture.get();
439                for (File f: openFileTsk.getSuccessfullyOpenedFiles()) {
440                    moveToDeletedLayersFolder(f);
441                }
442            } catch (InterruptedException | ExecutionException e) {
443                Logging.error(e);
444            }
445        });
446    }
447
448    /**
449     * Move file to the deleted layers directory.
450     * If moving does not work, it will try to delete the file directly.
451     * Afterwards, if the number of deleted layers gets larger than PROP_DELETED_LAYERS,
452     * some files in the deleted layers directory will be removed.
453     *
454     * @param f the file, usually from the autosave dir
455     */
456    private void moveToDeletedLayersFolder(File f) {
457        File backupFile = new File(deletedLayersDir, f.getName());
458        File pidFile = getPidFile(f);
459
460        if (backupFile.exists()) {
461            deletedLayers.remove(backupFile);
462            Utils.deleteFile(backupFile, marktr("Unable to delete old backup file {0}"));
463        }
464        if (f.renameTo(backupFile)) {
465            deletedLayers.add(backupFile);
466            Utils.deleteFile(pidFile);
467        } else {
468            Logging.warn(String.format("Could not move autosaved file %s to %s folder", f.getName(), deletedLayersDir.getName()));
469            // we cannot move to deleted folder, so just try to delete it directly
470            if (Utils.deleteFile(f, marktr("Unable to delete backup file {0}"))) {
471                Utils.deleteFile(pidFile, marktr("Unable to delete PID file {0}"));
472            }
473        }
474        while (deletedLayers.size() > PROP_DELETED_LAYERS.get()) {
475            File next = deletedLayers.remove();
476            if (next == null) {
477                break;
478            }
479            Utils.deleteFile(next, marktr("Unable to delete archived backup file {0}"));
480        }
481    }
482
483    /**
484     * Mark all unsaved layers as deleted. They are still preserved in the deleted layers folder.
485     */
486    public void discardUnsavedLayers() {
487        for (File f: getUnsavedLayersFiles()) {
488            moveToDeletedLayersFolder(f);
489        }
490    }
491}