Febbweiss/logstash-forwarder-java
1
0
Fork 0
mirror of https://github.com/Febbweiss/logstash-forwarder-java.git synced 2026-03-04 22:25:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implemented compressed file detection.

Browse Source
This commit is contained in:
didfet
2015-03-21 19:30:20 +01:00
parent 2a0146bbc7
commit fd03e33922
2 changed files with 64 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
src/main/java/info/fetter/logstashforwarder/FileReader.java
View File

@@ -1,20 +1,5 @@
package info.fetter.logstashforwarder;
import info.fetter.logstashforwarder.util.AdapterException;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.log4j.Logger;
/*
* Copyright 2015 Didier Fetter
*
@@ -32,8 +17,29 @@ import org.apache.log4j.Logger;
*
*/
import info.fetter.logstashforwarder.util.AdapterException;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.log4j.Logger;
public class FileReader {
private static Logger logger = Logger.getLogger(FileReader.class);
private static final byte[] ZIP_MAGIC = new byte[] {(byte) 0x50, (byte) 0x4b, (byte) 0x03, (byte) 0x04};
private static final byte[] LZW_MAGIC = new byte[] {(byte) 0x1f, (byte) 0x9d};
private static final byte[] LZH_MAGIC = new byte[] {(byte) 0x1f, (byte) 0xa0};
private static final byte[] GZ_MAGIC = new byte[] {(byte) 0x1f, (byte) 0x8b, (byte) 0x08};
private static final byte[][] MAGICS = new byte[][] {ZIP_MAGIC, LZW_MAGIC, LZH_MAGIC, GZ_MAGIC};
private ProtocolAdapter adapter;
private int spoolSize = 0;
private List<Event> eventList;
@@ -79,11 +85,36 @@ public class FileReader {
logger.trace("File : " + file + " pointer : " + pointer);
logger.trace("Space left in spool : " + spaceLeftInSpool);
}
pointer = readLines(state, spaceLeftInSpool);
if(isCompressedFile(state)) {
pointer = file.length();
} else {
pointer = readLines(state, spaceLeftInSpool);
}
pointerMap.put(file, pointer);
return eventList.size() - eventListSizeBefore; // Return number of events read
}
private boolean isCompressedFile(FileState state) {
RandomAccessFile reader = state.getRandomAccessFile();
try {
for(byte[] magic : MAGICS) {
byte[] fileBytes = new byte[magic.length];
reader.seek(0);
int read = reader.read(fileBytes);
if (read != magic.length) {
continue;
}
if(Arrays.equals(magic, fileBytes)) {
return true;
}
}
} catch(IOException e) {
logger.warn("Exception raised while reading file : " + state.getFile());
e.printStackTrace();
}
return false;
}
private long readLines(FileState state, int spaceLeftInSpool) {
RandomAccessFile reader = state.getRandomAccessFile();
long pos = state.getPointer();

34
src/main/java/info/fetter/logstashforwarder/Forwarder.java
View File

@@ -1,5 +1,22 @@
package info.fetter.logstashforwarder;
/*
* Copyright 2015 Didier Fetter
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
import static org.apache.log4j.Level.*;
import java.io.IOException;
@@ -24,23 +41,6 @@ import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.log4j.spi.RootLogger;
/*
* Copyright 2015 Didier Fetter
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
public class Forwarder {
private static Logger logger = Logger.getLogger(Forwarder.class);
private static int spoolSize = 1024;