001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.gui.layer;
003
004import static org.openstreetmap.josm.tools.I18n.marktr;
005import static org.openstreetmap.josm.tools.I18n.tr;
006import static org.openstreetmap.josm.tools.Utils.getSystemProperty;
007
008import java.io.BufferedReader;
009import java.io.File;
010import java.io.FileFilter;
011import java.io.IOException;
012import java.io.PrintStream;
013import java.lang.management.ManagementFactory;
014import java.nio.charset.StandardCharsets;
015import java.nio.file.Files;
016import java.nio.file.Path;
017import java.util.ArrayList;
018import java.util.Date;
019import java.util.Deque;
020import java.util.HashSet;
021import java.util.Iterator;
022import java.util.LinkedList;
023import java.util.List;
024import java.util.Set;
025import java.util.Timer;
026import java.util.TimerTask;
027import java.util.concurrent.ExecutionException;
028import java.util.concurrent.Future;
029import java.util.concurrent.TimeUnit;
030import java.util.regex.Pattern;
031
032import org.openstreetmap.josm.actions.OpenFileAction.OpenFileTask;
033import org.openstreetmap.josm.data.osm.DataSet;
034import org.openstreetmap.josm.data.osm.NoteData;
035import org.openstreetmap.josm.data.osm.NoteData.NoteDataUpdateListener;
036import org.openstreetmap.josm.data.osm.event.AbstractDatasetChangedEvent;
037import org.openstreetmap.josm.data.osm.event.DataSetListenerAdapter;
038import org.openstreetmap.josm.data.osm.event.DataSetListenerAdapter.Listener;
039import org.openstreetmap.josm.data.preferences.BooleanProperty;
040import org.openstreetmap.josm.data.preferences.IntegerProperty;
041import org.openstreetmap.josm.gui.MainApplication;
042import org.openstreetmap.josm.gui.Notification;
043import org.openstreetmap.josm.gui.io.importexport.NoteExporter;
044import org.openstreetmap.josm.gui.io.importexport.NoteImporter;
045import org.openstreetmap.josm.gui.io.importexport.OsmExporter;
046import org.openstreetmap.josm.gui.io.importexport.OsmImporter;
047import org.openstreetmap.josm.gui.layer.LayerManager.LayerAddEvent;
048import org.openstreetmap.josm.gui.layer.LayerManager.LayerChangeListener;
049import org.openstreetmap.josm.gui.layer.LayerManager.LayerOrderChangeEvent;
050import org.openstreetmap.josm.gui.layer.LayerManager.LayerRemoveEvent;
051import org.openstreetmap.josm.gui.util.GuiHelper;
052import org.openstreetmap.josm.spi.preferences.Config;
053import org.openstreetmap.josm.tools.Logging;
054import org.openstreetmap.josm.tools.Utils;
055
056/**
057 * Saves data and note layers periodically so they can be recovered in case of a crash.
058 *
059 * There are 2 directories
060 *  - autosave dir: copies of the currently open data layers are saved here every
061 *      PROP_INTERVAL seconds. When a data layer is closed normally, the corresponding
062 *      files are removed. If this dir is non-empty on start, JOSM assumes
063 *      that it crashed last time.
064 *  - deleted layers dir: "secondary archive" - when autosaved layers are restored
065 *      they are copied to this directory. We cannot keep them in the autosave folder,
066 *      but just deleting it would be dangerous: Maybe a feature inside the file
067 *      caused JOSM to crash. If the data is valuable, the user can still try to
068 *      open with another versions of JOSM or fix the problem manually.
069 *
070 *      The deleted layers dir keeps at most PROP_DELETED_LAYERS files.
071 *
072 * @since  3378 (creation)
073 * @since 10386 (new LayerChangeListener interface)
074 */
075public class AutosaveTask extends TimerTask implements LayerChangeListener, Listener, NoteDataUpdateListener {
076
077    private static final char[] ILLEGAL_CHARACTERS = {'/', '\n', '\r', '\t', '\0', '\f', '`', '?', '*', '\\', '<', '>', '|', '\"', ':'};
078    private static final String AUTOSAVE_DIR = "autosave";
079    private static final String DELETED_LAYERS_DIR = "autosave/deleted_layers";
080
081    /**
082     * If autosave is enabled
083     */
084    public static final BooleanProperty PROP_AUTOSAVE_ENABLED = new BooleanProperty("autosave.enabled", true);
085    /**
086     * The number of files to store per layer
087     */
088    public static final IntegerProperty PROP_FILES_PER_LAYER = new IntegerProperty("autosave.filesPerLayer", 1);
089    /**
090     * How many deleted layers should be stored
091     */
092    public static final IntegerProperty PROP_DELETED_LAYERS = new IntegerProperty("autosave.deletedLayersBackupCount", 5);
093    /**
094     * The autosave interval, in seconds
095     */
096    public static final IntegerProperty PROP_INTERVAL = new IntegerProperty("autosave.interval", (int) TimeUnit.MINUTES.toSeconds(5));
097    /**
098     * The maximum number of autosave files to store
099     */
100    public static final IntegerProperty PROP_INDEX_LIMIT = new IntegerProperty("autosave.index-limit", 1000);
101    /**
102     * Defines if a notification should be displayed after each autosave
103     */
104    public static final BooleanProperty PROP_NOTIFICATION = new BooleanProperty("autosave.notification", false);
105
106    protected static final class AutosaveLayerInfo<T extends AbstractModifiableLayer> {
107        private final T layer;
108        private String layerName;
109        private String layerFileName;
110        private final Deque<File> backupFiles = new LinkedList<>();
111
112        AutosaveLayerInfo(T layer) {
113            this.layer = layer;
114        }
115    }
116
117    private final DataSetListenerAdapter datasetAdapter = new DataSetListenerAdapter(this);
118    private final Set<DataSet> changedDatasets = new HashSet<>();
119    private final Set<NoteData> changedNoteData = new HashSet<>();
120    private final List<AutosaveLayerInfo<?>> layersInfo = new ArrayList<>();
121    private final Object layersLock = new Object();
122    private final Deque<File> deletedLayers = new LinkedList<>();
123
124    private final File autosaveDir = new File(Config.getDirs().getUserDataDirectory(true), AUTOSAVE_DIR);
125    private final File deletedLayersDir = new File(Config.getDirs().getUserDataDirectory(true), DELETED_LAYERS_DIR);
126
127    /**
128     * Replies the autosave directory.
129     * @return the autosave directory
130     * @since 10299
131     */
132    public final Path getAutosaveDir() {
133        return autosaveDir.toPath();
134    }
135
136    /**
137     * Starts the autosave background task.
138     */
139    public void schedule() {
140        if (PROP_INTERVAL.get() > 0) {
141
142            if (!autosaveDir.exists() && !autosaveDir.mkdirs()) {
143                Logging.warn(tr("Unable to create directory {0}, autosave will be disabled", autosaveDir.getAbsolutePath()));
144                return;
145            }
146            if (!deletedLayersDir.exists() && !deletedLayersDir.mkdirs()) {
147                Logging.warn(tr("Unable to create directory {0}, autosave will be disabled", deletedLayersDir.getAbsolutePath()));
148                return;
149            }
150
151            File[] files = deletedLayersDir.listFiles();
152            if (files != null) {
153                for (File f: files) {
154                    deletedLayers.add(f); // FIXME: sort by mtime
155                }
156            }
157
158            new Timer(true).schedule(this, TimeUnit.SECONDS.toMillis(1), TimeUnit.SECONDS.toMillis(PROP_INTERVAL.get()));
159            MainApplication.getLayerManager().addAndFireLayerChangeListener(this);
160        }
161    }
162
163    private static String getFileName(String layerName, int index) {
164        String result = layerName;
165        for (char illegalCharacter : ILLEGAL_CHARACTERS) {
166            result = result.replaceAll(Pattern.quote(String.valueOf(illegalCharacter)),
167                    '&' + String.valueOf((int) illegalCharacter) + ';');
168        }
169        if (index != 0) {
170            result = result + '_' + index;
171        }
172        return result;
173    }
174
175    private void setLayerFileName(AutosaveLayerInfo<?> layer) {
176        int index = 0;
177        while (true) {
178            String filename = getFileName(layer.layer.getName(), index);
179            boolean foundTheSame = false;
180            for (AutosaveLayerInfo<?> info: layersInfo) {
181                if (info != layer && filename.equals(info.layerFileName)) {
182                    foundTheSame = true;
183                    break;
184                }
185            }
186
187            if (!foundTheSame) {
188                layer.layerFileName = filename;
189                return;
190            }
191
192            index++;
193        }
194    }
195
196    protected File getNewLayerFile(AutosaveLayerInfo<?> layer, Date now, int startIndex) {
197        int index = startIndex;
198        while (true) {
199            String filename = String.format("%1$s_%2$tY%2$tm%2$td_%2$tH%2$tM%2$tS%2$tL%3$s",
200                    layer.layerFileName, now, index == 0 ? "" : ('_' + Integer.toString(index)));
201            File result = new File(autosaveDir, filename + '.' +
202                    (layer.layer instanceof NoteLayer ?
203                            Config.getPref().get("autosave.notes.extension", "osn") :
204                            Config.getPref().get("autosave.extension", "osm")));
205            try {
206                if (index > PROP_INDEX_LIMIT.get())
207                    throw new IOException("index limit exceeded");
208                if (result.createNewFile()) {
209                    createNewPidFile(autosaveDir, filename);
210                    return result;
211                } else {
212                    Logging.warn(tr("Unable to create file {0}, other filename will be used", result.getAbsolutePath()));
213                }
214            } catch (IOException e) {
215                Logging.log(Logging.LEVEL_ERROR, tr("IOError while creating file, autosave will be skipped: {0}", e.getMessage()), e);
216                return null;
217            }
218            index++;
219        }
220    }
221
222    private static void createNewPidFile(File autosaveDir, String filename) {
223        File pidFile = new File(autosaveDir, filename+".pid");
224        try (PrintStream ps = new PrintStream(pidFile, "UTF-8")) {
225            ps.println(ManagementFactory.getRuntimeMXBean().getName());
226        } catch (IOException | SecurityException t) {
227            Logging.error(t);
228        }
229    }
230
231    private void savelayer(AutosaveLayerInfo<?> info) {
232        if (!info.layer.getName().equals(info.layerName)) {
233            setLayerFileName(info);
234            info.layerName = info.layer.getName();
235        }
236        try {
237            if (info.layer instanceof OsmDataLayer) {
238                OsmDataLayer dataLayer = (OsmDataLayer) info.layer;
239                if (changedDatasets.remove(dataLayer.data)) {
240                    File file = getNewLayerFile(info, new Date(), 0);
241                    if (file != null) {
242                        info.backupFiles.add(file);
243                        new OsmExporter().exportData(file, info.layer, true /* no backup with appended ~ */);
244                    }
245                }
246            } else if (info.layer instanceof NoteLayer) {
247                NoteLayer noteLayer = (NoteLayer) info.layer;
248                if (changedNoteData.remove(noteLayer.getNoteData())) {
249                    File file = getNewLayerFile(info, new Date(), 0);
250                    if (file != null) {
251                        info.backupFiles.add(file);
252                        new NoteExporter().exportData(file, info.layer);
253                    }
254                }
255            }
256        } catch (IOException e) {
257            Logging.error(e);
258        }
259        while (info.backupFiles.size() > PROP_FILES_PER_LAYER.get()) {
260            File oldFile = info.backupFiles.remove();
261            if (Utils.deleteFile(oldFile, marktr("Unable to delete old backup file {0}"))) {
262                Utils.deleteFile(getPidFile(oldFile), marktr("Unable to delete old backup file {0}"));
263            }
264        }
265    }
266
267    @Override
268    public void run() {
269        synchronized (layersLock) {
270            try {
271                for (AutosaveLayerInfo<?> info: layersInfo) {
272                    savelayer(info);
273                }
274                changedDatasets.clear();
275                changedNoteData.clear();
276                if (PROP_NOTIFICATION.get() && !layersInfo.isEmpty()) {
277                    GuiHelper.runInEDT(this::displayNotification);
278                }
279            } catch (RuntimeException t) { // NOPMD
280                // Don't let exception stop time thread
281                Logging.error("Autosave failed:");
282                Logging.error(t);
283            }
284        }
285    }
286
287    protected void displayNotification() {
288        new Notification(tr("Your work has been saved automatically."))
289        .setDuration(Notification.TIME_SHORT)
290        .show();
291    }
292
293    @Override
294    public void layerOrderChanged(LayerOrderChangeEvent e) {
295        // Do nothing
296    }
297
298    private void registerNewlayer(OsmDataLayer layer) {
299        synchronized (layersLock) {
300            layer.getDataSet().addDataSetListener(datasetAdapter);
301            layersInfo.add(new AutosaveLayerInfo<>(layer));
302        }
303    }
304
305    private void registerNewlayer(NoteLayer layer) {
306        synchronized (layersLock) {
307            layer.getNoteData().addNoteDataUpdateListener(this);
308            layersInfo.add(new AutosaveLayerInfo<>(layer));
309        }
310    }
311
312    @Override
313    public void layerAdded(LayerAddEvent e) {
314        if (e.getAddedLayer() instanceof OsmDataLayer) {
315            registerNewlayer((OsmDataLayer) e.getAddedLayer());
316        } else if (e.getAddedLayer() instanceof NoteLayer) {
317            registerNewlayer((NoteLayer) e.getAddedLayer());
318        }
319    }
320
321    @Override
322    public void layerRemoving(LayerRemoveEvent e) {
323        if (e.getRemovedLayer() instanceof OsmDataLayer) {
324            synchronized (layersLock) {
325                OsmDataLayer osmLayer = (OsmDataLayer) e.getRemovedLayer();
326                osmLayer.getDataSet().removeDataSetListener(datasetAdapter);
327                cleanupLayer(osmLayer);
328            }
329        } else if (e.getRemovedLayer() instanceof NoteLayer) {
330            synchronized (layersLock) {
331                NoteLayer noteLayer = (NoteLayer) e.getRemovedLayer();
332                noteLayer.getNoteData().removeNoteDataUpdateListener(this);
333                cleanupLayer(noteLayer);
334            }
335        }
336    }
337
338    private void cleanupLayer(AbstractModifiableLayer removedLayer) {
339        Iterator<AutosaveLayerInfo<?>> it = layersInfo.iterator();
340        while (it.hasNext()) {
341            AutosaveLayerInfo<?> info = it.next();
342            if (info.layer == removedLayer) {
343
344                savelayer(info);
345                File lastFile = info.backupFiles.pollLast();
346                if (lastFile != null) {
347                    moveToDeletedLayersFolder(lastFile);
348                }
349                for (File file: info.backupFiles) {
350                    if (Utils.deleteFile(file)) {
351                        Utils.deleteFile(getPidFile(file));
352                    }
353                }
354
355                it.remove();
356            }
357        }
358    }
359
360    @Override
361    public void processDatasetEvent(AbstractDatasetChangedEvent event) {
362        changedDatasets.add(event.getDataset());
363    }
364
365    @Override
366    public void noteDataUpdated(NoteData data) {
367        changedNoteData.add(data);
368    }
369
370    @Override
371    public void selectedNoteChanged(NoteData noteData) {
372        // Do nothing
373    }
374
375    protected File getPidFile(File osmFile) {
376        return new File(autosaveDir, osmFile.getName().replaceFirst("[.][^.]+$", ".pid"));
377    }
378
379    /**
380     * Replies the list of .osm files still present in autosave dir, that are not currently managed by another instance of JOSM.
381     * These files are hence unsaved layers from an old instance of JOSM that crashed and may be recovered by this instance.
382     * @return The list of .osm files still present in autosave dir, that are not currently managed by another instance of JOSM
383     */
384    public List<File> getUnsavedLayersFiles() {
385        List<File> result = new ArrayList<>();
386        try {
387            File[] files = autosaveDir.listFiles((FileFilter)
388                    pathname -> OsmImporter.FILE_FILTER.accept(pathname) || NoteImporter.FILE_FILTER.accept(pathname));
389            if (files == null)
390                return result;
391            for (File file: files) {
392                if (file.isFile()) {
393                    boolean skipFile = false;
394                    File pidFile = getPidFile(file);
395                    if (pidFile.exists()) {
396                        try (BufferedReader reader = Files.newBufferedReader(pidFile.toPath(), StandardCharsets.UTF_8)) {
397                            String jvmId = reader.readLine();
398                            if (jvmId != null) {
399                                String pid = jvmId.split("@")[0];
400                                skipFile = jvmPerfDataFileExists(pid);
401                            }
402                        } catch (IOException | SecurityException t) {
403                            Logging.error(t);
404                        }
405                    }
406                    if (!skipFile) {
407                        result.add(file);
408                    }
409                }
410            }
411        } catch (SecurityException e) {
412            Logging.log(Logging.LEVEL_ERROR, "Unable to list unsaved layers files", e);
413        }
414        return result;
415    }
416
417    private static boolean jvmPerfDataFileExists(final String jvmId) {
418        File jvmDir = new File(getSystemProperty("java.io.tmpdir") + File.separator + "hsperfdata_" + getSystemProperty("user.name"));
419        if (jvmDir.exists() && jvmDir.canRead()) {
420            File[] files = jvmDir.listFiles((FileFilter) file -> file.getName().equals(jvmId) && file.isFile());
421            return files != null && files.length == 1;
422        }
423        return false;
424    }
425
426    /**
427     * Recover the unsaved layers and open them asynchronously.
428     * @return A future that can be used to wait for the completion of this task.
429     */
430    public Future<?> recoverUnsavedLayers() {
431        List<File> files = getUnsavedLayersFiles();
432        final OpenFileTask openFileTsk = new OpenFileTask(files, null, tr("Restoring files"));
433        final Future<?> openFilesFuture = MainApplication.worker.submit(openFileTsk);
434        return MainApplication.worker.submit(() -> {
435            try {
436                // Wait for opened tasks to be generated.
437                openFilesFuture.get();
438                for (File f: openFileTsk.getSuccessfullyOpenedFiles()) {
439                    moveToDeletedLayersFolder(f);
440                }
441            } catch (InterruptedException | ExecutionException e) {
442                Logging.error(e);
443            }
444        });
445    }
446
447    /**
448     * Move file to the deleted layers directory.
449     * If moving does not work, it will try to delete the file directly.
450     * Afterwards, if the number of deleted layers gets larger than PROP_DELETED_LAYERS,
451     * some files in the deleted layers directory will be removed.
452     *
453     * @param f the file, usually from the autosave dir
454     */
455    private void moveToDeletedLayersFolder(File f) {
456        File backupFile = new File(deletedLayersDir, f.getName());
457        File pidFile = getPidFile(f);
458
459        if (backupFile.exists()) {
460            deletedLayers.remove(backupFile);
461            Utils.deleteFile(backupFile, marktr("Unable to delete old backup file {0}"));
462        }
463        if (f.renameTo(backupFile)) {
464            deletedLayers.add(backupFile);
465            Utils.deleteFile(pidFile);
466        } else {
467            Logging.warn(String.format("Could not move autosaved file %s to %s folder", f.getName(), deletedLayersDir.getName()));
468            // we cannot move to deleted folder, so just try to delete it directly
469            if (Utils.deleteFile(f, marktr("Unable to delete backup file {0}"))) {
470                Utils.deleteFile(pidFile, marktr("Unable to delete PID file {0}"));
471            }
472        }
473        while (deletedLayers.size() > PROP_DELETED_LAYERS.get()) {
474            File next = deletedLayers.remove();
475            if (next == null) {
476                break;
477            }
478            Utils.deleteFile(next, marktr("Unable to delete archived backup file {0}"));
479        }
480    }
481
482    /**
483     * Mark all unsaved layers as deleted. They are still preserved in the deleted layers folder.
484     */
485    public void discardUnsavedLayers() {
486        for (File f: getUnsavedLayersFiles()) {
487            moveToDeletedLayersFolder(f);
488        }
489    }
490}