diff --git a/.travis.yml b/.travis.yml index 8a8e9e1e05..6408621539 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,7 @@ language: java sudo: false script: mvn clean verify jdk: -- oraclejdk8 +- openjdk8 after_script: - bash ./deploy_site.sh @@ -25,7 +25,7 @@ deploy: on: tags: true repo: fact-project/fact-tools - java: oraclejdk8 + java: openjdk8 notifications: slack: diff --git a/CHANGELOG.md b/CHANGELOG.md index 89ebafa068..804d5e902a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog for the fact-tools +# Version 1.1.2 -- 11.10.2019 + +* Better error message for broken FITS files +* Allow only skipping empty files in FileListMultiStream + # Version 1.1.2 -- 17.12.2018 * Fix reading of uncompressed values from zfits heap tiles (Issue #389) diff --git a/examples/forErna/std_analysis_erna_simulations.xml b/examples/forErna/std_analysis_erna_simulations.xml index c604a86f47..9ba2f6a047 100644 --- a/examples/forErna/std_analysis_erna_simulations.xml +++ b/examples/forErna/std_analysis_erna_simulations.xml @@ -14,7 +14,7 @@ - + diff --git a/pom.xml b/pom.xml index 086395ec38..ecf9c2959c 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ de.sfb876 fact-tools fact-tools - 1.1.2 + 1.1.3 http://sfb876.de/fact-tools/ diff --git a/src/main/java/fact/io/FactFileListMultiStream.java b/src/main/java/fact/io/FactFileListMultiStream.java index 6456f89229..7f0ada4aa3 100644 --- a/src/main/java/fact/io/FactFileListMultiStream.java +++ b/src/main/java/fact/io/FactFileListMultiStream.java @@ -2,6 +2,9 @@ import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; +import fact.io.hdureader.FITSStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import stream.Data; import stream.annotations.Parameter; import stream.io.AbstractStream; @@ -43,6 +46,8 @@ */ public class FactFileListMultiStream extends AbstractMultiStream { + Logger log = LoggerFactory.getLogger(FactFileListMultiStream.class); + private DataDrsPair dataDrsPair; public FactFileListMultiStream(SourceURL url) { @@ -67,16 +72,19 @@ public DataDrsPair(String dataFile, String drsFile) { @Parameter(required = true, description = "A file containing a json array of dicts with the paths to the files.") - private SourceURL url = null; + public SourceURL url = null; @Parameter(required = false, description = "Flag indicating whether next file should be tried in case of errors in underlying stream.", defaultValue = "false") - private boolean skipErrors = false; + public boolean skipErrors = false; + + @Parameter(required = false, description = "Similar to skipErrors but will only skip the MissingHDU exception, i.e. empty files", defaultValue = "false") + public boolean skipEmpty = false; @Parameter(required = false, defaultValue = "drs_path") - private String drsPathKey = "drs_path"; + public String drsPathKey = "drs_path"; @Parameter(required = false, defaultValue = "data_path") - private String dataPathKey = "data_path"; + public String dataPathKey = "data_path"; //counts how many files have been processed private int filesCounter = 0; @@ -91,6 +99,9 @@ public DataDrsPair(String dataFile, String drsFile) { */ @Override public void init() throws Exception { + log.info("Skipping empty files: {}", skipEmpty); + log.info("Skipping broken files: {}", skipErrors); + if (!fileQueue.isEmpty()) { log.debug("files already loaded"); return; @@ -155,11 +166,22 @@ public Data readNext() throws Exception { data.put("@drsFile", dataDrsPair.drsFile); return data; + + } catch (FITSStream.MissingHDUException e) { + log.info("File: {} does not contain the Events HDU", stream.getUrl()); + if (skipEmpty) { + log.warn("Skipping empty file {}. Continuing with next file.", stream.getUrl()); + stream = null; + return this.readNext(); + } else { + log.error("Stopping stream because of missing HDU in one input file"); + stream.close(); + throw new RuntimeException(e); + } } catch (IOException e) { log.info("File: " + stream.getUrl().toString() + " throws IOException."); - if (skipErrors) { - log.info("Skipping broken files. Continuing with next file."); + log.warn("Skipping broken file {}. Continuing with next file.", stream.getUrl()); e.printStackTrace(); stream = null; return this.readNext(); @@ -177,20 +199,9 @@ public void close() throws Exception { log.info("In total {} files were processed.", filesCounter); } + + @Override public void setUrl(SourceURL url) { this.url = url; } - - public void setDrsPathKey(String drsPathKey) { - this.drsPathKey = drsPathKey; - } - - public void setDataPathKey(String dataPathKey) { - this.dataPathKey = dataPathKey; - } - - public void setSkipErrors(boolean skipErrors) { - this.skipErrors = skipErrors; - } - } diff --git a/src/main/java/fact/io/hdureader/FITS.java b/src/main/java/fact/io/hdureader/FITS.java index 6b66227be5..304b4e9e53 100644 --- a/src/main/java/fact/io/hdureader/FITS.java +++ b/src/main/java/fact/io/hdureader/FITS.java @@ -102,7 +102,11 @@ public FITS(URL url) throws IOException { } } catch (EOFException e) { - primaryHDU = hdus.get(0); + if (hdus.size() < 1) { + log.warn("File '{}' does not contain any HDUs", url); + } else { + primaryHDU = hdus.get(0); + } stream.close(); log.debug("A total of {} HDUs were found in the file.", hdus.size()); diff --git a/src/main/java/fact/io/hdureader/FITSStream.java b/src/main/java/fact/io/hdureader/FITSStream.java index 56aac8138c..f23c03606c 100644 --- a/src/main/java/fact/io/hdureader/FITSStream.java +++ b/src/main/java/fact/io/hdureader/FITSStream.java @@ -72,7 +72,7 @@ public void init() throws Exception { } // create a refrence to the events hdu - eventHDU = fits.getHDU(nameHDU).orElseThrow(() -> new IOException("Inputfile did not contain HDU '" + nameHDU + "'")); + eventHDU = fits.getHDU(nameHDU).orElseThrow(() -> new MissingHDUException("Inputfile did not contain HDU '" + nameHDU + "'")); BinTable eventsTable = eventHDU.getBinTable(); @@ -137,4 +137,10 @@ public Data readNext() throws Exception { public void skipRows(int amount) throws IOException{ reader.skipRows(amount); } + + public class MissingHDUException extends IOException { + public MissingHDUException(String errorMessage) { + super(errorMessage); + } + } } diff --git a/src/test/java/fact/io/FITSStreamTest.java b/src/test/java/fact/io/FITSStreamTest.java index 39de52a690..800646e365 100644 --- a/src/test/java/fact/io/FITSStreamTest.java +++ b/src/test/java/fact/io/FITSStreamTest.java @@ -61,4 +61,12 @@ public void testFitsKeys() { } } + @Test(expected=FITSStream.MissingHDUException.class) + public void testEmptyFile() throws Exception { + URL u = FITSStreamTest.class.getResource("/empty_fits.fits.gz"); + SourceURL url = new SourceURL(u); + FITSStream stream = new FITSStream(url); + stream.init(); + } + } diff --git a/src/test/java/fact/io/ListMultiStreamTest.java b/src/test/java/fact/io/ListMultiStreamTest.java index 88d876ecd9..b9793263e6 100644 --- a/src/test/java/fact/io/ListMultiStreamTest.java +++ b/src/test/java/fact/io/ListMultiStreamTest.java @@ -4,12 +4,13 @@ import org.junit.Test; import stream.Data; import stream.io.SourceURL; +import stream.shell.Run; import java.io.File; import java.net.URL; import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.*; /** * Created by kaibrugge on 14.04.15. @@ -35,12 +36,45 @@ public void testDrsInjection() throws Exception { FITSStream m = new FITSStream(); multiStream.addStream("test", m); + multiStream.skipEmpty = true; multiStream.init(); Data data = multiStream.readNext(); + assertTrue(multiStream.skipEmpty); File drsFile = (File) data.get("@drsFile"); assertThat(drsFile.getName(), is("20140920_66.drs.fits")); } + + @Test + public void testNotSkipEmpty () throws Exception { + URL u = FITSStreamTest.class.getResource("/dummy_files/file_list_empty.json"); + + FactFileListMultiStream multiStream = new FactFileListMultiStream(new SourceURL(u)); + multiStream.setUrl(new SourceURL(u)); + FITSStream m = new FITSStream(); + multiStream.addStream("test", m); + multiStream.init(); + + try { + Data data = multiStream.readNext(); + fail("Runtime exception expected"); + } catch (RuntimeException e) {} + + } + + @Test + public void testSkipEmpty () throws Exception { + URL u = FITSStreamTest.class.getResource("/dummy_files/file_list_empty.json"); + + FactFileListMultiStream multiStream = new FactFileListMultiStream(new SourceURL(u)); + multiStream.skipEmpty = true; + multiStream.setUrl(new SourceURL(u)); + FITSStream m = new FITSStream(); + multiStream.addStream("test", m); + multiStream.init(); + + Data data = multiStream.readNext(); + } } diff --git a/src/test/resources/dummy_files/file_list_empty.json b/src/test/resources/dummy_files/file_list_empty.json new file mode 100644 index 0000000000..1cd82b641c --- /dev/null +++ b/src/test/resources/dummy_files/file_list_empty.json @@ -0,0 +1,4 @@ +[ + {"drs_path":"src\/main\/resources\/dummy_files\/20140920_66.drs.fits","later":"src\/main\/resources\/20140920_66.drs.fits","night":"20140920","data_path":"src\/test\/resources\/empty_fits.fits.gz"}, + {"drs_path":"src\/main\/resources\/dummy_files\/20140920_66.drs.fits","later":"src\/main\/resources\/20140920_66.drs.fits","night":"20140920","data_path":"src\/main\/resources\/testDataFile.fits.gz"} +] \ No newline at end of file diff --git a/src/test/resources/empty_fits.fits.gz b/src/test/resources/empty_fits.fits.gz new file mode 100644 index 0000000000..7f15f29bf8 Binary files /dev/null and b/src/test/resources/empty_fits.fits.gz differ