Febbweiss/logstash-forwarder-java
1
0
Fork 0
mirror of https://github.com/Febbweiss/logstash-forwarder-java.git synced 2026-03-05 06:35:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added multiline support for files.

Browse Source
This commit is contained in:
toliman7
2016-06-12 18:22:06 +02:00
parent b6e2b943ca
commit bd0c146a89
11 changed files with 293 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/main/java/info/fetter/logstashforwarder/FileModificationListener.java
View File

@@ -25,10 +25,12 @@ import org.apache.commons.io.monitor.FileAlterationObserver;
public class FileModificationListener implements FileAlterationListener {
private Event fields;
private FileWatcher watcher;
private Multiline multiline;
public FileModificationListener(FileWatcher watcher, Event fields) {
public FileModificationListener(FileWatcher watcher, Event fields, Multiline multiline) {
this.watcher = watcher;
this.fields = fields;
this.multiline = multiline;
}
public void onDirectoryChange(File file) {
@@ -44,11 +46,11 @@ public class FileModificationListener implements FileAlterationListener {
}
public void onFileChange(File file) {
watcher.onFileChange(file, fields);
watcher.onFileChange(file, fields, multiline);
}
public void onFileCreate(File file) {
watcher.onFileCreate(file, fields);
watcher.onFileCreate(file, fields, multiline);
}
public void onFileDelete(File file) {

46
src/main/java/info/fetter/logstashforwarder/FileReader.java
View File

@@ -26,6 +26,7 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import org.apache.commons.lang.ArrayUtils;
import org.apache.log4j.Logger;
@@ -123,19 +124,62 @@ public class FileReader extends Reader {
private long readLines(FileState state, int spaceLeftInSpool) {
RandomAccessFile reader = state.getRandomAccessFile();
long pos = state.getPointer();
Multiline multiline = state.getMultiline();
try {
reader.seek(pos);
byte[] line = readLine(reader);
byte[] bufferedLines = null;
while (line != null && spaceLeftInSpool > 0) {
if(logger.isTraceEnabled()) {
logger.trace("-- Read line : " + new String(line));
logger.trace("-- Space left in spool : " + spaceLeftInSpool);
}
pos = reader.getFilePointer();
addEvent(state, pos, line);
if (multiline == null) {
addEvent(state, pos, line);
}
else {
if (logger.isTraceEnabled()) {
logger.trace("-- Multiline : " + multiline);
logger.trace("-- Multiline : matches " + multiline.isPatternFound(line));
}
if (multiline.isPatternFound(line))
{
// buffer the line
if (bufferedLines != null)
{
bufferedLines = ArrayUtils.addAll(bufferedLines, line);
}
else
{
bufferedLines = line;
}
}
else {
if (multiline.isPrevious()) {
// did not match, so new event started
if (bufferedLines != null) {
addEvent(state, pos, bufferedLines);
}
bufferedLines = line;
}
else {
// did not match, add the current line
if (bufferedLines != null) {
addEvent(state, pos, ArrayUtils.addAll(bufferedLines, line));
bufferedLines = null;
}
else
addEvent(state, pos, line);
}
}
}
line = readLine(reader);
spaceLeftInSpool--;
}
if (bufferedLines != null) {
addEvent(state, pos, bufferedLines); // send any buffered lines left
}
reader.seek(pos); // Ensure we can re-read if necessary
} catch(IOException e) {
logger.warn("Exception raised while reading file : " + state.getFile(), e);

21
src/main/java/info/fetter/logstashforwarder/FileState.java
View File

@@ -25,6 +25,7 @@ import java.io.RandomAccessFile;
import org.apache.commons.lang.builder.ToStringBuilder;
import com.fasterxml.jackson.annotation.JsonIgnore;
import java.util.Map;
public class FileState {
@JsonIgnore
@@ -48,6 +49,10 @@ public class FileState {
private FileState oldFileState;
@JsonIgnore
private Event fields;
@JsonIgnore
private Multiline multiline;
@JsonIgnore
private byte[] bufferedLines = null;
public FileState() {
}
@@ -172,6 +177,22 @@ public class FileState {
public void setFields(Event fields) {
this.fields = fields;
}
public Multiline getMultiline() {
return multiline;
}
public void setMultiline(Multiline multiline) {
this.multiline = multiline;
}
public byte[] getBufferedLines() {
return bufferedLines;
}
public void setBufferedLines(byte[] bufferedLines) {
this.bufferedLines = bufferedLines;
}
@Override
public String toString() {

31
src/main/java/info/fetter/logstashforwarder/FileWatcher.java
View File

@@ -76,14 +76,14 @@ public class FileWatcher {
printWatchMap();
}
public void addFilesToWatch(String fileToWatch, Event fields, int deadTime) {
public void addFilesToWatch(String fileToWatch, Event fields, int deadTime, Multiline multiline) {
try {
if(fileToWatch.equals("-")) {
addStdIn(fields);
} else if(fileToWatch.contains("*")) {
addWildCardFiles(fileToWatch, fields, deadTime);
addWildCardFiles(fileToWatch, fields, deadTime, multiline);
} else {
addSingleFile(fileToWatch, fields, deadTime);
addSingleFile(fileToWatch, fields, deadTime, multiline);
}
} catch(Exception e) {
throw new RuntimeException(e);
@@ -219,7 +219,7 @@ public class FileWatcher {
removeMarkedFilesFromWatchMap();
}
private void addSingleFile(String fileToWatch, Event fields, int deadTime) throws Exception {
private void addSingleFile(String fileToWatch, Event fields, int deadTime, Multiline multiline) throws Exception {
logger.info("Watching file : " + new File(fileToWatch).getCanonicalPath());
String directory = FilenameUtils.getFullPath(fileToWatch);
String fileName = FilenameUtils.getName(fileToWatch);
@@ -227,10 +227,10 @@ public class FileWatcher {
FileFilterUtils.fileFileFilter(),
FileFilterUtils.nameFileFilter(fileName),
new LastModifiedFileFilter(deadTime));
initializeWatchMap(new File(directory), fileFilter, fields);
initializeWatchMap(new File(directory), fileFilter, fields, multiline);
}
private void addWildCardFiles(String filesToWatch, Event fields, int deadTime) throws Exception {
private void addWildCardFiles(String filesToWatch, Event fields, int deadTime, Multiline multiline) throws Exception {
logger.info("Watching wildcard files : " + filesToWatch);
String directory = FilenameUtils.getFullPath(filesToWatch);
String wildcard = FilenameUtils.getName(filesToWatch);
@@ -239,7 +239,7 @@ public class FileWatcher {
FileFilterUtils.fileFileFilter(),
new WildcardFileFilter(wildcard),
new LastModifiedFileFilter(deadTime));
initializeWatchMap(new File(directory), fileFilter, fields);
initializeWatchMap(new File(directory), fileFilter, fields, multiline);
}
private void addStdIn(Event fields) {
@@ -248,22 +248,22 @@ public class FileWatcher {
stdinConfigured = true;
}
private void initializeWatchMap(File directory, IOFileFilter fileFilter, Event fields) throws Exception {
private void initializeWatchMap(File directory, IOFileFilter fileFilter, Event fields, Multiline multiline) throws Exception {
if(!directory.isDirectory()) {
logger.warn("Directory " + directory + " does not exist");
return;
}
FileAlterationObserver observer = new FileAlterationObserver(directory, fileFilter);
FileModificationListener listener = new FileModificationListener(this, fields);
FileModificationListener listener = new FileModificationListener(this, fields, multiline);
observer.addListener(listener);
observerList.add(observer);
observer.initialize();
for(File file : FileUtils.listFiles(directory, fileFilter, null)) {
addFileToWatchMap(newWatchMap, file, fields);
addFileToWatchMap(newWatchMap, file, fields, multiline);
}
}
private void addFileToWatchMap(Map<File,FileState> map, File file, Event fields) {
private void addFileToWatchMap(Map<File,FileState> map, File file, Event fields, Multiline multiline) {
try {
FileState state = new FileState(file);
state.setFields(fields);
@@ -272,25 +272,26 @@ public class FileWatcher {
long signature = FileSigner.computeSignature(state.getRandomAccessFile(), signatureLength);
state.setSignature(signature);
logger.trace("Setting signature of size : " + signatureLength + " on file : " + file + " : " + signature);
state.setMultiline(multiline);
map.put(file, state);
} catch(IOException e) {
logger.error("Caught IOException : " + e.getMessage());
}
}
public void onFileChange(File file, Event fields) {
public void onFileChange(File file, Event fields, Multiline multiline) {
try {
logger.debug("Change detected on file : " + file.getCanonicalPath());
addFileToWatchMap(newWatchMap, file, fields);
addFileToWatchMap(newWatchMap, file, fields, multiline);
} catch (IOException e) {
logger.error("Caught IOException : " + e.getMessage());
}
}
public void onFileCreate(File file, Event fields) {
public void onFileCreate(File file, Event fields, Multiline multiline) {
try {
logger.debug("Create detected on file : " + file.getCanonicalPath());
addFileToWatchMap(newWatchMap, file, fields);
addFileToWatchMap(newWatchMap, file, fields, multiline);
} catch (IOException e) {
logger.error("Caught IOException : " + e.getMessage());
}

2
src/main/java/info/fetter/logstashforwarder/Forwarder.java
View File

@@ -80,7 +80,7 @@ public class Forwarder {
configManager.readConfiguration();
for(FilesSection files : configManager.getConfig().getFiles()) {
for(String path : files.getPaths()) {
watcher.addFilesToWatch(path, new Event(files.getFields()), files.getDeadTimeInSeconds() * 1000);
watcher.addFilesToWatch(path, new Event(files.getFields()), files.getDeadTimeInSeconds() * 1000, files.getMultiline());
}
}
watcher.initialize();

89
src/main/java/info/fetter/logstashforwarder/Multiline.java Normal file
View File

@@ -0,0 +1,89 @@
package info.fetter.logstashforwarder;
/*
* Copyright 2015 Didier Fetter
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
import java.io.UnsupportedEncodingException;
import java.util.Map;
import java.util.regex.Pattern;
import org.apache.commons.lang.builder.ToStringBuilder;
public class Multiline {
public enum WhatType { Previous, Next };
private Pattern pattern = null;
private boolean negate = false;
private WhatType what = WhatType.Previous;
public Multiline() {
}
public Multiline(Multiline event) {
if(event != null) {
this.negate = event.negate;
this.pattern = event.pattern;
this.what = event.what;
}
}
public Multiline(Map<String,String> fields) throws UnsupportedEncodingException {
String strPattern = "";
for(String key : fields.keySet()) {
if ("pattern".equals(key))
strPattern = fields.get(key);
else if ("negate".equals(key))
negate = Boolean.parseBoolean(fields.get(key));
else if ("what".equals(key))
what = WhatType.valueOf(fields.get(key));
else
throw new UnsupportedEncodingException(key + " not supported");
}
pattern = Pattern.compile(strPattern);
}
public Pattern getPattern() {
return pattern;
}
public boolean isNegate() {
return negate;
}
public WhatType getWhat() {
return what;
}
public boolean isPrevious() {
return what == WhatType.Previous;
}
public boolean isPatternFound (byte[] line) {
boolean result = pattern.matcher(new String(line)).find();
if (negate) return !result;
return result;
}
@Override
public String toString() {
return new ToStringBuilder(this).
append("pattern", pattern).
append("negate", negate).
append("what", what).
toString();
}
}

13
src/main/java/info/fetter/logstashforwarder/config/FilesSection.java
View File

@@ -19,16 +19,20 @@ package info.fetter.logstashforwarder.config;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
import org.apache.commons.lang.builder.ToStringBuilder;
import com.fasterxml.jackson.annotation.JsonProperty;
import info.fetter.logstashforwarder.Multiline;
import java.io.UnsupportedEncodingException;
public class FilesSection {
private List<String> paths;
private Map<String,String> fields;
@JsonProperty("dead time")
private String deadTime = "24h";
private Multiline multiline;
public List<String> getPaths() {
return paths;
@@ -78,13 +82,22 @@ public class FilesSection {
public void setDeadTime(String deadTime) {
this.deadTime = deadTime;
}
public Multiline getMultiline() {
return multiline;
}
public void setMultiline(Map<String, String> multilineMap) throws UnsupportedEncodingException {
this.multiline = new Multiline(multilineMap);
}
@Override
public String toString() {
return new ToStringBuilder(this).
append("paths", paths).
append("fields", fields).
append("dead time", deadTime).
append("multiline", multiline).
toString();
}
}