mirror of
https://github.com/Febbweiss/logstash-forwarder-java.git
synced 2026-03-04 22:25:39 +00:00
Implemented FileReader.
This commit is contained in:
@@ -23,6 +23,15 @@ import java.util.Map;
|
|||||||
public class Event {
|
public class Event {
|
||||||
private Map<String,byte[]> keyValues = new HashMap<String,byte[]>(10);
|
private Map<String,byte[]> keyValues = new HashMap<String,byte[]>(10);
|
||||||
|
|
||||||
|
public Event() {
|
||||||
|
}
|
||||||
|
|
||||||
|
public Event(Event event) {
|
||||||
|
if(event != null) {
|
||||||
|
keyValues.putAll(event.keyValues);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public void addField(String key, byte[] value) {
|
public void addField(String key, byte[] value) {
|
||||||
keyValues.put(key, value);
|
keyValues.put(key, value);
|
||||||
}
|
}
|
||||||
@@ -31,6 +40,10 @@ public class Event {
|
|||||||
keyValues.put(key, value.getBytes());
|
keyValues.put(key, value.getBytes());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void addField(String key, long value) {
|
||||||
|
keyValues.put(key, String.valueOf(value).getBytes());
|
||||||
|
}
|
||||||
|
|
||||||
public Map<String,byte[]> getKeyValues() {
|
public Map<String,byte[]> getKeyValues() {
|
||||||
return keyValues;
|
return keyValues;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,14 @@
|
|||||||
package info.fetter.logstashforwarder;
|
package info.fetter.logstashforwarder;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.RandomAccessFile;
|
||||||
|
import java.net.InetAddress;
|
||||||
|
import java.net.UnknownHostException;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
|
import java.util.HashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Copyright 2015 Didier Fetter
|
* Copyright 2015 Didier Fetter
|
||||||
@@ -25,24 +32,88 @@ public class FileReader {
|
|||||||
private ProtocolAdapter adapter;
|
private ProtocolAdapter adapter;
|
||||||
private int spoolSize = DEFAULT_SPOOL_SIZE;
|
private int spoolSize = DEFAULT_SPOOL_SIZE;
|
||||||
private List<Event> eventList;
|
private List<Event> eventList;
|
||||||
|
private Map<File,Long> pointerMap;
|
||||||
|
private String hostname;
|
||||||
|
{
|
||||||
|
try {
|
||||||
|
hostname = InetAddress.getLocalHost().getHostName();
|
||||||
|
} catch (UnknownHostException e) {
|
||||||
|
throw new RuntimeException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public FileReader(int spoolSize) {
|
public FileReader(int spoolSize) {
|
||||||
this.spoolSize = spoolSize;
|
this.spoolSize = spoolSize;
|
||||||
eventList = new ArrayList<Event>(spoolSize);
|
eventList = new ArrayList<Event>(spoolSize);
|
||||||
}
|
}
|
||||||
|
|
||||||
public int readFiles(List<FileState> fileList) {
|
public int readFiles(List<FileState> fileList) throws IOException {
|
||||||
// TODO: Read files and send events until there's nothing left to read or spool size reached
|
int eventCount = 0;
|
||||||
int eventCounter = 0;
|
pointerMap = new HashMap<File,Long>(fileList.size(),1);
|
||||||
for(FileState state : fileList) {
|
for(FileState state : fileList) {
|
||||||
eventCounter += readFile(state, spoolSize - eventCounter);
|
eventCount += readFile(state, spoolSize - eventCount);
|
||||||
}
|
}
|
||||||
return 0; // Return number of events sent to adapter
|
adapter.sendEvents(eventList);
|
||||||
|
for(FileState state : fileList) {
|
||||||
|
state.setPointer(pointerMap.get(state.getFile()));
|
||||||
|
}
|
||||||
|
eventList.clear();
|
||||||
|
return eventCount; // Return number of events sent to adapter
|
||||||
}
|
}
|
||||||
|
|
||||||
private int readFile(FileState state, int spaceLeftInSpool) {
|
private int readFile(FileState state, int spaceLeftInSpool) throws IOException {
|
||||||
// TODO Read file
|
int eventListSizeBefore = eventList.size();
|
||||||
return 0; // Return number of events read
|
File file = state.getFile();
|
||||||
|
long pointer = state.getPointer();
|
||||||
|
RandomAccessFile reader = state.getRandomAccessFile();
|
||||||
|
reader.seek(pointer);
|
||||||
|
pointer = readLines(state, spaceLeftInSpool);
|
||||||
|
pointerMap.put(file, pointer);
|
||||||
|
return eventList.size() - eventListSizeBefore; // Return number of events read
|
||||||
|
}
|
||||||
|
|
||||||
|
private long readLines(FileState state, int spaceLeftInSpool) throws IOException {
|
||||||
|
RandomAccessFile reader = state.getRandomAccessFile();
|
||||||
|
long pos = reader.getFilePointer();
|
||||||
|
String line = readLine(reader);
|
||||||
|
while (line != null && spaceLeftInSpool > 0) {
|
||||||
|
pos = reader.getFilePointer();
|
||||||
|
addEvent(state, pos, line);
|
||||||
|
line = readLine(reader);
|
||||||
|
}
|
||||||
|
reader.seek(pos); // Ensure we can re-read if necessary
|
||||||
|
return pos;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String readLine(RandomAccessFile reader) throws IOException {
|
||||||
|
StringBuffer sb = new StringBuffer();
|
||||||
|
int ch;
|
||||||
|
boolean seenCR = false;
|
||||||
|
while((ch=reader.read()) != -1) {
|
||||||
|
switch(ch) {
|
||||||
|
case '\n':
|
||||||
|
return sb.toString();
|
||||||
|
case '\r':
|
||||||
|
seenCR = true;
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
if (seenCR) {
|
||||||
|
sb.append('\r');
|
||||||
|
seenCR = false;
|
||||||
|
}
|
||||||
|
sb.append((char)ch); // add character, not its ascii value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void addEvent(FileState state, long pos, String line) throws IOException {
|
||||||
|
Event event = new Event(state.getFields());
|
||||||
|
event.addField("file", state.getFile().getCanonicalPath());
|
||||||
|
event.addField("offset", pos);
|
||||||
|
event.addField("line", line);
|
||||||
|
event.addField("host", hostname);
|
||||||
|
eventList.add(event);
|
||||||
}
|
}
|
||||||
|
|
||||||
public ProtocolAdapter getAdapter() {
|
public ProtocolAdapter getAdapter() {
|
||||||
|
|||||||
@@ -34,6 +34,7 @@ public class FileState {
|
|||||||
private long pointer = 0;
|
private long pointer = 0;
|
||||||
private FileState oldFileState;
|
private FileState oldFileState;
|
||||||
private String fileName;
|
private String fileName;
|
||||||
|
private Event fields;
|
||||||
|
|
||||||
public FileState(File file) throws IOException {
|
public FileState(File file) throws IOException {
|
||||||
this.file = file;
|
this.file = file;
|
||||||
@@ -116,4 +117,12 @@ public class FileState {
|
|||||||
return fileName;
|
return fileName;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public Event getFields() {
|
||||||
|
return fields;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setFields(Event fields) {
|
||||||
|
this.fields = fields;
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user