From 3a10652d9f8137915fdd27be8bf78e3574248add Mon Sep 17 00:00:00 2001 From: didfet Date: Tue, 10 Mar 2015 23:23:19 +0100 Subject: [PATCH] Added FileWatcher and LastModifiedFileFilter. --- .../fetter/logstashforwarder/FileState.java | 55 +++++++++++ .../fetter/logstashforwarder/FileWatcher.java | 91 ++++++++++++------- .../util/LastModifiedFileFilter.java | 48 ++++++++++ 3 files changed, 160 insertions(+), 34 deletions(-) create mode 100644 src/main/java/info/fetter/logstashforwarder/FileState.java create mode 100644 src/main/java/info/fetter/logstashforwarder/util/LastModifiedFileFilter.java diff --git a/src/main/java/info/fetter/logstashforwarder/FileState.java b/src/main/java/info/fetter/logstashforwarder/FileState.java new file mode 100644 index 0000000..d99964a --- /dev/null +++ b/src/main/java/info/fetter/logstashforwarder/FileState.java @@ -0,0 +1,55 @@ +package info.fetter.logstashforwarder; + +/* + * Copyright 2015 Didier Fetter + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import java.io.File; +import java.io.IOException; + +public class FileState { + private File file; + private String filePath; + private long lastModified; + private long size; + + public FileState(File file) throws IOException { + this.file = file; + filePath = file.getCanonicalPath(); + } + + public void refresh() { + lastModified = file.lastModified(); + size = file.length(); + } + + public File getFile() { + return file; + } + + public long getLastModified() { + return lastModified; + } + + public long getSize() { + return size; + } + + public String getFilePath() { + return filePath; + } + +} diff --git a/src/main/java/info/fetter/logstashforwarder/FileWatcher.java b/src/main/java/info/fetter/logstashforwarder/FileWatcher.java index 0f4f3c9..ee53547 100644 --- a/src/main/java/info/fetter/logstashforwarder/FileWatcher.java +++ b/src/main/java/info/fetter/logstashforwarder/FileWatcher.java @@ -1,17 +1,5 @@ package info.fetter.logstashforwarder; -import java.io.File; -import java.util.ArrayList; -import java.util.List; - -import org.apache.commons.io.FilenameUtils; -import org.apache.commons.io.filefilter.FileFilterUtils; -import org.apache.commons.io.filefilter.IOFileFilter; -import org.apache.commons.io.filefilter.WildcardFileFilter; -import org.apache.commons.io.monitor.FileAlterationListener; -import org.apache.commons.io.monitor.FileAlterationObserver; -import org.apache.log4j.Logger; - /* * Copyright 2015 Didier Fetter * @@ -29,16 +17,35 @@ import org.apache.log4j.Logger; * */ +import info.fetter.logstashforwarder.util.LastModifiedFileFilter; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.io.filefilter.FileFilterUtils; +import org.apache.commons.io.filefilter.IOFileFilter; +import org.apache.commons.io.filefilter.WildcardFileFilter; +import org.apache.commons.io.monitor.FileAlterationListener; +import org.apache.commons.io.monitor.FileAlterationObserver; +import org.apache.log4j.Logger; + public class FileWatcher implements FileAlterationListener { private static final Logger logger = Logger.getLogger(FileWatcher.class); private List observerList = new ArrayList(); - static final int ONE_DAY = 24 * 3600 * 1000; - private long cutoff; - + private static final int ONE_DAY = 24 * 3600 * 1000; + private long deadTime; + private Map watchMap = new HashMap(); + public FileWatcher(long deadTime) { - cutoff = System.currentTimeMillis() - deadTime; + this.deadTime = deadTime; } - + public FileWatcher() { this(ONE_DAY); } @@ -58,40 +65,56 @@ public class FileWatcher implements FileAlterationListener { } private void watchFile(String fileToWatch) throws Exception { - logger.info("Watching file : " + new File(fileToWatch).getAbsolutePath()); + logger.info("Watching file : " + new File(fileToWatch).getCanonicalPath()); String directory = FilenameUtils.getFullPath(fileToWatch); String fileName = FilenameUtils.getName(fileToWatch); IOFileFilter fileFilter = FileFilterUtils.and( FileFilterUtils.fileFileFilter(), FileFilterUtils.nameFileFilter(fileName), - FileFilterUtils.ageFileFilter(cutoff, false)); - FileAlterationObserver observer = new FileAlterationObserver(new File(directory), fileFilter); - observer.addListener(this); - observerList.add(observer); - observer.initialize(); + new LastModifiedFileFilter(deadTime)); + updateWatchMap(new File(directory), fileFilter); } private void watchWildCardFiles(String filesToWatch) throws Exception { logger.info("Watching wildcard files : " + filesToWatch); String directory = FilenameUtils.getFullPath(filesToWatch); String wildcard = FilenameUtils.getName(filesToWatch); - logger.trace("Directory : " + new File(directory).getAbsolutePath() + ", wildcard : " + wildcard); + logger.trace("Directory : " + new File(directory).getCanonicalPath() + ", wildcard : " + wildcard); IOFileFilter fileFilter = FileFilterUtils.and( FileFilterUtils.fileFileFilter(), new WildcardFileFilter(wildcard), - FileFilterUtils.ageFileFilter(cutoff, false)); - FileAlterationObserver observer = new FileAlterationObserver(new File(directory), fileFilter); - observer.addListener(this); - observerList.add(observer); - observer.initialize(); + new LastModifiedFileFilter(deadTime)); + updateWatchMap(new File(directory), fileFilter); } - - private void watchStdIn() { logger.info("Watching stdin : not implemented yet"); } + private void updateWatchMap(File directory, IOFileFilter fileFilter) throws Exception { + FileAlterationObserver observer = new FileAlterationObserver(directory, fileFilter); + observer.addListener(this); + observerList.add(observer); + observer.initialize(); + for(File file : FileUtils.listFiles(directory, fileFilter, null)) { + addFileToWatchMap(file); + } + } + + private void addFileToWatchMap(File file) { + try { + FileState state = new FileState(file); + state.refresh(); + watchMap.put(file, state); + } catch(IOException e) { + logger.error("Caught IOException : " + e.getMessage()); + } + } + + private void removeFileFromWatchMap(File file) { + watchMap.remove(file); + } + public void onDirectoryChange(File directory) { // Do nothing } @@ -109,21 +132,21 @@ public class FileWatcher implements FileAlterationListener { } public void onFileCreate(File file) { - logger.debug("Create detected on file : " + file.getAbsolutePath()); + logger.debug("Create detected on file : " + file.getAbsolutePath()); + addFileToWatchMap(file); } public void onFileDelete(File file) { logger.debug("Delete detected on file : " + file.getAbsolutePath()); + removeFileFromWatchMap(file); } public void onStart(FileAlterationObserver observer) { // TODO Auto-generated method stub - } public void onStop(FileAlterationObserver observer) { // TODO Auto-generated method stub - } public void checkFiles() { diff --git a/src/main/java/info/fetter/logstashforwarder/util/LastModifiedFileFilter.java b/src/main/java/info/fetter/logstashforwarder/util/LastModifiedFileFilter.java new file mode 100644 index 0000000..7142490 --- /dev/null +++ b/src/main/java/info/fetter/logstashforwarder/util/LastModifiedFileFilter.java @@ -0,0 +1,48 @@ +package info.fetter.logstashforwarder.util; + +/* + * Copyright 2015 Didier Fetter + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import java.io.File; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.filefilter.AbstractFileFilter; + +public class LastModifiedFileFilter extends AbstractFileFilter { + private boolean after; + private long cutoff; + + public LastModifiedFileFilter(long cutoff, boolean after) { + this.after = after; + this.cutoff = cutoff; + } + + public LastModifiedFileFilter(long cutoff) { + this(cutoff, true); + } + + @Override + public boolean accept(File file) { + long timeMillis = System.currentTimeMillis() - cutoff; + if(after) { + return FileUtils.isFileNewer(file, timeMillis); + } else { + return FileUtils.isFileOlder(file, timeMillis); + } + } + +}