diff --git a/.travis.yml b/.travis.yml
index 8a8e9e1e05..6408621539 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -2,7 +2,7 @@ language: java
sudo: false
script: mvn clean verify
jdk:
-- oraclejdk8
+- openjdk8
after_script:
- bash ./deploy_site.sh
@@ -25,7 +25,7 @@ deploy:
on:
tags: true
repo: fact-project/fact-tools
- java: oraclejdk8
+ java: openjdk8
notifications:
slack:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 89ebafa068..804d5e902a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,10 @@
# Changelog for the fact-tools
+# Version 1.1.2 -- 11.10.2019
+
+* Better error message for broken FITS files
+* Allow only skipping empty files in FileListMultiStream
+
# Version 1.1.2 -- 17.12.2018
* Fix reading of uncompressed values from zfits heap tiles (Issue #389)
diff --git a/examples/forErna/std_analysis_erna_simulations.xml b/examples/forErna/std_analysis_erna_simulations.xml
index c604a86f47..9ba2f6a047 100644
--- a/examples/forErna/std_analysis_erna_simulations.xml
+++ b/examples/forErna/std_analysis_erna_simulations.xml
@@ -14,7 +14,7 @@
-
+
diff --git a/pom.xml b/pom.xml
index 086395ec38..ecf9c2959c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
de.sfb876
fact-tools
fact-tools
- 1.1.2
+ 1.1.3
http://sfb876.de/fact-tools/
diff --git a/src/main/java/fact/io/FactFileListMultiStream.java b/src/main/java/fact/io/FactFileListMultiStream.java
index 6456f89229..7f0ada4aa3 100644
--- a/src/main/java/fact/io/FactFileListMultiStream.java
+++ b/src/main/java/fact/io/FactFileListMultiStream.java
@@ -2,6 +2,9 @@
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
+import fact.io.hdureader.FITSStream;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import stream.Data;
import stream.annotations.Parameter;
import stream.io.AbstractStream;
@@ -43,6 +46,8 @@
*/
public class FactFileListMultiStream extends AbstractMultiStream {
+ Logger log = LoggerFactory.getLogger(FactFileListMultiStream.class);
+
private DataDrsPair dataDrsPair;
public FactFileListMultiStream(SourceURL url) {
@@ -67,16 +72,19 @@ public DataDrsPair(String dataFile, String drsFile) {
@Parameter(required = true, description = "A file containing a json array of dicts with the paths to the files.")
- private SourceURL url = null;
+ public SourceURL url = null;
@Parameter(required = false, description = "Flag indicating whether next file should be tried in case of errors in underlying stream.", defaultValue = "false")
- private boolean skipErrors = false;
+ public boolean skipErrors = false;
+
+ @Parameter(required = false, description = "Similar to skipErrors but will only skip the MissingHDU exception, i.e. empty files", defaultValue = "false")
+ public boolean skipEmpty = false;
@Parameter(required = false, defaultValue = "drs_path")
- private String drsPathKey = "drs_path";
+ public String drsPathKey = "drs_path";
@Parameter(required = false, defaultValue = "data_path")
- private String dataPathKey = "data_path";
+ public String dataPathKey = "data_path";
//counts how many files have been processed
private int filesCounter = 0;
@@ -91,6 +99,9 @@ public DataDrsPair(String dataFile, String drsFile) {
*/
@Override
public void init() throws Exception {
+ log.info("Skipping empty files: {}", skipEmpty);
+ log.info("Skipping broken files: {}", skipErrors);
+
if (!fileQueue.isEmpty()) {
log.debug("files already loaded");
return;
@@ -155,11 +166,22 @@ public Data readNext() throws Exception {
data.put("@drsFile", dataDrsPair.drsFile);
return data;
+
+ } catch (FITSStream.MissingHDUException e) {
+ log.info("File: {} does not contain the Events HDU", stream.getUrl());
+ if (skipEmpty) {
+ log.warn("Skipping empty file {}. Continuing with next file.", stream.getUrl());
+ stream = null;
+ return this.readNext();
+ } else {
+ log.error("Stopping stream because of missing HDU in one input file");
+ stream.close();
+ throw new RuntimeException(e);
+ }
} catch (IOException e) {
log.info("File: " + stream.getUrl().toString() + " throws IOException.");
-
if (skipErrors) {
- log.info("Skipping broken files. Continuing with next file.");
+ log.warn("Skipping broken file {}. Continuing with next file.", stream.getUrl());
e.printStackTrace();
stream = null;
return this.readNext();
@@ -177,20 +199,9 @@ public void close() throws Exception {
log.info("In total {} files were processed.", filesCounter);
}
+
+ @Override
public void setUrl(SourceURL url) {
this.url = url;
}
-
- public void setDrsPathKey(String drsPathKey) {
- this.drsPathKey = drsPathKey;
- }
-
- public void setDataPathKey(String dataPathKey) {
- this.dataPathKey = dataPathKey;
- }
-
- public void setSkipErrors(boolean skipErrors) {
- this.skipErrors = skipErrors;
- }
-
}
diff --git a/src/main/java/fact/io/hdureader/FITS.java b/src/main/java/fact/io/hdureader/FITS.java
index 6b66227be5..304b4e9e53 100644
--- a/src/main/java/fact/io/hdureader/FITS.java
+++ b/src/main/java/fact/io/hdureader/FITS.java
@@ -102,7 +102,11 @@ public FITS(URL url) throws IOException {
}
} catch (EOFException e) {
- primaryHDU = hdus.get(0);
+ if (hdus.size() < 1) {
+ log.warn("File '{}' does not contain any HDUs", url);
+ } else {
+ primaryHDU = hdus.get(0);
+ }
stream.close();
log.debug("A total of {} HDUs were found in the file.", hdus.size());
diff --git a/src/main/java/fact/io/hdureader/FITSStream.java b/src/main/java/fact/io/hdureader/FITSStream.java
index 56aac8138c..f23c03606c 100644
--- a/src/main/java/fact/io/hdureader/FITSStream.java
+++ b/src/main/java/fact/io/hdureader/FITSStream.java
@@ -72,7 +72,7 @@ public void init() throws Exception {
}
// create a refrence to the events hdu
- eventHDU = fits.getHDU(nameHDU).orElseThrow(() -> new IOException("Inputfile did not contain HDU '" + nameHDU + "'"));
+ eventHDU = fits.getHDU(nameHDU).orElseThrow(() -> new MissingHDUException("Inputfile did not contain HDU '" + nameHDU + "'"));
BinTable eventsTable = eventHDU.getBinTable();
@@ -137,4 +137,10 @@ public Data readNext() throws Exception {
public void skipRows(int amount) throws IOException{
reader.skipRows(amount);
}
+
+ public class MissingHDUException extends IOException {
+ public MissingHDUException(String errorMessage) {
+ super(errorMessage);
+ }
+ }
}
diff --git a/src/test/java/fact/io/FITSStreamTest.java b/src/test/java/fact/io/FITSStreamTest.java
index 39de52a690..800646e365 100644
--- a/src/test/java/fact/io/FITSStreamTest.java
+++ b/src/test/java/fact/io/FITSStreamTest.java
@@ -61,4 +61,12 @@ public void testFitsKeys() {
}
}
+ @Test(expected=FITSStream.MissingHDUException.class)
+ public void testEmptyFile() throws Exception {
+ URL u = FITSStreamTest.class.getResource("/empty_fits.fits.gz");
+ SourceURL url = new SourceURL(u);
+ FITSStream stream = new FITSStream(url);
+ stream.init();
+ }
+
}
diff --git a/src/test/java/fact/io/ListMultiStreamTest.java b/src/test/java/fact/io/ListMultiStreamTest.java
index 88d876ecd9..b9793263e6 100644
--- a/src/test/java/fact/io/ListMultiStreamTest.java
+++ b/src/test/java/fact/io/ListMultiStreamTest.java
@@ -4,12 +4,13 @@
import org.junit.Test;
import stream.Data;
import stream.io.SourceURL;
+import stream.shell.Run;
import java.io.File;
import java.net.URL;
import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.assertThat;
+import static org.junit.Assert.*;
/**
* Created by kaibrugge on 14.04.15.
@@ -35,12 +36,45 @@ public void testDrsInjection() throws Exception {
FITSStream m = new FITSStream();
multiStream.addStream("test", m);
+ multiStream.skipEmpty = true;
multiStream.init();
Data data = multiStream.readNext();
+ assertTrue(multiStream.skipEmpty);
File drsFile = (File) data.get("@drsFile");
assertThat(drsFile.getName(), is("20140920_66.drs.fits"));
}
+
+ @Test
+ public void testNotSkipEmpty () throws Exception {
+ URL u = FITSStreamTest.class.getResource("/dummy_files/file_list_empty.json");
+
+ FactFileListMultiStream multiStream = new FactFileListMultiStream(new SourceURL(u));
+ multiStream.setUrl(new SourceURL(u));
+ FITSStream m = new FITSStream();
+ multiStream.addStream("test", m);
+ multiStream.init();
+
+ try {
+ Data data = multiStream.readNext();
+ fail("Runtime exception expected");
+ } catch (RuntimeException e) {}
+
+ }
+
+ @Test
+ public void testSkipEmpty () throws Exception {
+ URL u = FITSStreamTest.class.getResource("/dummy_files/file_list_empty.json");
+
+ FactFileListMultiStream multiStream = new FactFileListMultiStream(new SourceURL(u));
+ multiStream.skipEmpty = true;
+ multiStream.setUrl(new SourceURL(u));
+ FITSStream m = new FITSStream();
+ multiStream.addStream("test", m);
+ multiStream.init();
+
+ Data data = multiStream.readNext();
+ }
}
diff --git a/src/test/resources/dummy_files/file_list_empty.json b/src/test/resources/dummy_files/file_list_empty.json
new file mode 100644
index 0000000000..1cd82b641c
--- /dev/null
+++ b/src/test/resources/dummy_files/file_list_empty.json
@@ -0,0 +1,4 @@
+[
+ {"drs_path":"src\/main\/resources\/dummy_files\/20140920_66.drs.fits","later":"src\/main\/resources\/20140920_66.drs.fits","night":"20140920","data_path":"src\/test\/resources\/empty_fits.fits.gz"},
+ {"drs_path":"src\/main\/resources\/dummy_files\/20140920_66.drs.fits","later":"src\/main\/resources\/20140920_66.drs.fits","night":"20140920","data_path":"src\/main\/resources\/testDataFile.fits.gz"}
+]
\ No newline at end of file
diff --git a/src/test/resources/empty_fits.fits.gz b/src/test/resources/empty_fits.fits.gz
new file mode 100644
index 0000000000..7f15f29bf8
Binary files /dev/null and b/src/test/resources/empty_fits.fits.gz differ