From 797f6f41bb52c0e2aa7a232aa523052e3d47033b Mon Sep 17 00:00:00 2001 From: didfet Date: Thu, 12 Mar 2015 23:46:51 +0100 Subject: [PATCH] FileWatcher debugging. --- .../fetter/logstashforwarder/FileState.java | 8 +-- .../fetter/logstashforwarder/FileWatcher.java | 60 ++++++++++--------- .../logstashforwarder/FileWatcherTest.java | 33 +++++++--- 3 files changed, 59 insertions(+), 42 deletions(-) diff --git a/src/main/java/info/fetter/logstashforwarder/FileState.java b/src/main/java/info/fetter/logstashforwarder/FileState.java index 6197dbe..10e902a 100644 --- a/src/main/java/info/fetter/logstashforwarder/FileState.java +++ b/src/main/java/info/fetter/logstashforwarder/FileState.java @@ -23,7 +23,7 @@ import java.io.RandomAccessFile; public class FileState { private File file; - private String filePath; + private String directory; private long lastModified; private long size; private boolean deleted = false; @@ -37,7 +37,7 @@ public class FileState { public FileState(File file) throws IOException { this.file = file; - filePath = file.getCanonicalPath(); + directory = file.getCanonicalFile().getParent(); fileName = file.getName(); randomAccessFile = new RandomAccessFile(file, "r"); lastModified = file.lastModified(); @@ -56,8 +56,8 @@ public class FileState { return size; } - public String getFilePath() { - return filePath; + public String getDirectory() { + return directory; } public boolean isDeleted() { diff --git a/src/main/java/info/fetter/logstashforwarder/FileWatcher.java b/src/main/java/info/fetter/logstashforwarder/FileWatcher.java index 0b7df96..7720404 100644 --- a/src/main/java/info/fetter/logstashforwarder/FileWatcher.java +++ b/src/main/java/info/fetter/logstashforwarder/FileWatcher.java @@ -68,7 +68,7 @@ public class FileWatcher implements FileAlterationListener { public void checkFiles() throws IOException { logger.debug("Checking files"); - printWatchMap(); + //printWatchMap(); for(FileAlterationObserver observer : observerList) { observer.checkAndNotify(); } @@ -78,14 +78,17 @@ public class FileWatcher implements FileAlterationListener { } private void processModifications() throws IOException { - removeMarkedFilesFromWatchMap(); for(File file : changedWatchMap.keySet()) { FileState state = changedWatchMap.get(file); logger.trace("Checking file : " + file.getCanonicalPath()); + logger.trace("-- Last modified : " + state.getLastModified()); + logger.trace("-- Size : " + state.getSize()); + logger.trace("-- Directory : " + state.getDirectory()); + logger.trace("-- Filename : " + state.getFileName()); // Determine if file is still the same - logger.trace("Determine if file name has not changed"); + logger.trace("Determine if file has just been written to"); FileState oldState = watchMap.get(file); if(oldState != null) { if(oldState.getSize() > state.getSize()) { @@ -96,6 +99,7 @@ public class FileWatcher implements FileAlterationListener { // File is the same state.setOldFileState(oldState); logger.trace("Same signature size and value : file is the same"); + continue; } else if(oldState.getSignatureLength() < state.getSignatureLength()){ // Compute the signature on the new file long signature = FileSigner.computeSignature(state.getRandomAccessFile(), oldState.getSignatureLength()); @@ -103,6 +107,7 @@ public class FileWatcher implements FileAlterationListener { // File is the same state.setOldFileState(oldState); logger.trace("Same signature : file is the same"); + continue; } else { // File can't be the same logger.trace("Signature different : file can't be the same"); @@ -114,36 +119,19 @@ public class FileWatcher implements FileAlterationListener { } } - // Determine if there was a file with the same size and last modification date in the same directory and with a different name - logger.trace("Determine if file has just been renamed"); + // Determine if file has been renamed and/or written to if(state.getOldFileState() == null) { - for(File otherFile : changedWatchMap.keySet()) { + logger.trace("Determine if file has been renamed and/or written to"); + for(File otherFile : watchMap.keySet()) { FileState otherState = watchMap.get(otherFile); - if(otherState != null - && state.getLastModified() == otherState.getLastModified() - && state.getSize() == otherState.getSize() - && state.getFilePath().equals(otherState.getFilePath()) - && ! state.getFileName().equals(otherState.getFileName())) { - logger.trace("Comparing to : " + otherFile.getCanonicalPath()); - // Assume file has been renamed - state.setOldFileState(otherState); - logger.trace("Same directory, same size and last modification date : file has been renamed to : " + otherFile.getCanonicalPath()); - } - } - } - - // Determine if file has been renamed and appended - logger.trace("Determine if file has been renamed and appended"); - if(state.getOldFileState() == null) { - for(File otherFile : changedWatchMap.keySet()) { - FileState otherState = watchMap.get(otherFile); - if(otherState != null && state.getSize() > otherState.getSize() && state.getFilePath().equals(otherState.getFilePath())) { + if(otherState != null && state.getSize() >= otherState.getSize() && state.getDirectory().equals(otherState.getDirectory())) { logger.trace("Comparing to : " + otherFile.getCanonicalPath()); // File in the same directory which was smaller if(otherState.getSignatureLength() == state.getSignatureLength() && otherState.getSignature() == state.getSignature()) { // File is the same state.setOldFileState(otherState); logger.trace("Same signature size and value : file is the same"); + break; } else if(otherState.getSignatureLength() < state.getSignatureLength()){ // Compute the signature on the new file long signature = FileSigner.computeSignature(state.getRandomAccessFile(), otherState.getSignatureLength()); @@ -151,6 +139,7 @@ public class FileWatcher implements FileAlterationListener { // File is the same state.setOldFileState(otherState); logger.trace("Same signature : file is the same"); + break; } else { // File can't be the same logger.trace("Signature different : file can't be the same"); @@ -165,7 +154,7 @@ public class FileWatcher implements FileAlterationListener { } // Refresh file state - logger.trace("refreshing file state"); + logger.trace("Refreshing file state"); for(File file : changedWatchMap.keySet()) { logger.trace("Refreshing file : " + file.getCanonicalPath()); FileState state = changedWatchMap.get(file); @@ -182,11 +171,15 @@ public class FileWatcher implements FileAlterationListener { // Replacing old state logger.trace("Replacing old state"); for(File file : changedWatchMap.keySet()) { - logger.trace("Replacing file : " + file.getCanonicalPath()); + //logger.trace("Replacing file : " + file.getCanonicalPath()); FileState state = changedWatchMap.get(file); watchMap.put(file, state); } + // Truncating changedWatchMap + changedWatchMap.clear(); + + removeMarkedFilesFromWatchMap(); } private void watchFile(String fileToWatch) throws Exception { @@ -272,7 +265,7 @@ public class FileWatcher implements FileAlterationListener { logger.trace("WatchMap contents : "); for(File file : watchMap.keySet()) { FileState state = watchMap.get(file); - logger.trace("\tFile : " + state.getFilePath() + " marked for deletion : " + state.isDeleted()); + logger.trace("\tFile : " + state.getDirectory() + " marked for deletion : " + state.isDeleted()); } } } @@ -293,10 +286,18 @@ public class FileWatcher implements FileAlterationListener { for(File file : markedList) { FileState state = watchMap.remove(file); state.getRandomAccessFile().close(); - logger.trace("\tFile : " + state.getFilePath() + " removed"); + logger.trace("\tFile : " + file.getCanonicalFile() + " removed"); } } } + + public void close() throws IOException { + logger.debug("Closing all files"); + for(File file : watchMap.keySet()) { + FileState state = watchMap.get(file); + state.getRandomAccessFile().close(); + } + } public void onDirectoryChange(File directory) { // Do nothing @@ -317,4 +318,5 @@ public class FileWatcher implements FileAlterationListener { public void onStop(FileAlterationObserver observer) { // TODO Auto-generated method stub } + } diff --git a/src/test/java/info/fetter/logstashforwarder/FileWatcherTest.java b/src/test/java/info/fetter/logstashforwarder/FileWatcherTest.java index b985de0..4f767e9 100644 --- a/src/test/java/info/fetter/logstashforwarder/FileWatcherTest.java +++ b/src/test/java/info/fetter/logstashforwarder/FileWatcherTest.java @@ -47,20 +47,35 @@ public class FileWatcherTest { File file3 = new File("test3.txt"); File file4 = new File("test4.txt"); + File testDir = new File("test"); + //FileUtils.forceMkdir(new File("test")); + watcher.checkFiles(); - Thread.sleep(500); - FileUtils.write(file1, "line 1\n", true); - Thread.sleep(500); + Thread.sleep(100); + FileUtils.write(file1, "file 1 line 1\n", true); + Thread.sleep(100); watcher.checkFiles(); - FileUtils.forceDeleteOnExit(file1); -// FileUtils.touch(file2); -// Thread.sleep(500); -// watcher.checkFiles(); + FileUtils.write(file1, "file 1 line 2\n", true); + FileUtils.write(file2, "file 2 line 1\n", true); + Thread.sleep(1000); + watcher.checkFiles(); + FileUtils.moveFileToDirectory(file1, testDir, true); + FileUtils.write(file2, "file 2 line 2\n", true); + FileUtils.moveFile(file2, file1); + FileUtils.write(file2, "file 3 line 1\n", true); // FileUtils.touch(file3); // FileUtils.forceDelete(file1); // FileUtils.forceDelete(file2); -// Thread.sleep(500); -// watcher.checkFiles(); + Thread.sleep(1000); + watcher.checkFiles(); + + + watcher.close(); + FileUtils.forceDelete(file1); + FileUtils.forceDelete(file2); + FileUtils.forceDelete(testDir); + + // FileUtils.moveFile(file3, file4); // FileUtils.touch(file3); // Thread.sleep(500);