From 1f765a7fb61969f6ddcff55780b0e5eb5ad9774d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20N=C3=B6the?= Date: Fri, 11 Oct 2019 11:26:48 +0200 Subject: [PATCH 1/4] Better error message for empty FITS files, allow only skipping empty files in FileListMultiStream --- .../forErna/std_analysis_erna_simulations.xml | 2 +- .../java/fact/io/FactFileListMultiStream.java | 46 ++++++++++++++---- src/main/java/fact/io/hdureader/FITS.java | 6 ++- .../java/fact/io/hdureader/FITSStream.java | 8 ++- src/test/java/fact/io/FITSStreamTest.java | 8 +++ src/test/resources/empty_fits.fits.gz | Bin 0 -> 2967 bytes 6 files changed, 58 insertions(+), 12 deletions(-) create mode 100644 src/test/resources/empty_fits.fits.gz diff --git a/examples/forErna/std_analysis_erna_simulations.xml b/examples/forErna/std_analysis_erna_simulations.xml index c604a86f47..9ba2f6a047 100644 --- a/examples/forErna/std_analysis_erna_simulations.xml +++ b/examples/forErna/std_analysis_erna_simulations.xml @@ -14,7 +14,7 @@ - + diff --git a/src/main/java/fact/io/FactFileListMultiStream.java b/src/main/java/fact/io/FactFileListMultiStream.java index 6456f89229..0b983b304f 100644 --- a/src/main/java/fact/io/FactFileListMultiStream.java +++ b/src/main/java/fact/io/FactFileListMultiStream.java @@ -2,6 +2,9 @@ import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; +import fact.io.hdureader.FITSStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import stream.Data; import stream.annotations.Parameter; import stream.io.AbstractStream; @@ -43,6 +46,8 @@ */ public class FactFileListMultiStream extends AbstractMultiStream { + Logger log = LoggerFactory.getLogger(FactFileListMultiStream.class); + private DataDrsPair dataDrsPair; public FactFileListMultiStream(SourceURL url) { @@ -67,16 +72,19 @@ public DataDrsPair(String dataFile, String drsFile) { @Parameter(required = true, description = "A file containing a json array of dicts with the paths to the files.") - private SourceURL url = null; + public SourceURL url = null; @Parameter(required = false, description = "Flag indicating whether next file should be tried in case of errors in underlying stream.", defaultValue = "false") - private boolean skipErrors = false; + public boolean skipErrors = false; + + @Parameter(required = false, description = "Similar to skipErrors but will only skip the MissingHDU exception, i.e. empty files", defaultValue = "false") + public boolean skipEmpty = false; @Parameter(required = false, defaultValue = "drs_path") - private String drsPathKey = "drs_path"; + public String drsPathKey = "drs_path"; @Parameter(required = false, defaultValue = "data_path") - private String dataPathKey = "data_path"; + public String dataPathKey = "data_path"; //counts how many files have been processed private int filesCounter = 0; @@ -91,6 +99,9 @@ public DataDrsPair(String dataFile, String drsFile) { */ @Override public void init() throws Exception { + log.info("Skipping empty files: {}", skipEmpty); + log.info("Skipping broken files: {}", skipErrors); + if (!fileQueue.isEmpty()) { log.debug("files already loaded"); return; @@ -155,6 +166,18 @@ public Data readNext() throws Exception { data.put("@drsFile", dataDrsPair.drsFile); return data; + + } catch (FITSStream.MissingHDUException e) { + log.info("File: {} does not contain the Events HDU", stream.getUrl()); + if (skipEmpty) { + log.info("Skipping empty file. Continuing with next file."); + stream = null; + return this.readNext(); + } else { + log.error("Stopping stream because of missing HDU in one input file"); + stream.close(); + throw new RuntimeException(e); + } } catch (IOException e) { log.info("File: " + stream.getUrl().toString() + " throws IOException."); @@ -177,10 +200,20 @@ public void close() throws Exception { log.info("In total {} files were processed.", filesCounter); } + + @Override public void setUrl(SourceURL url) { this.url = url; } + public void setSkipErrors(boolean skipErrors) { + this.skipErrors = skipErrors; + } + + public void setSkipEmpty(boolean skipEmpty) { + this.skipEmpty = skipEmpty; + } + public void setDrsPathKey(String drsPathKey) { this.drsPathKey = drsPathKey; } @@ -188,9 +221,4 @@ public void setDrsPathKey(String drsPathKey) { public void setDataPathKey(String dataPathKey) { this.dataPathKey = dataPathKey; } - - public void setSkipErrors(boolean skipErrors) { - this.skipErrors = skipErrors; - } - } diff --git a/src/main/java/fact/io/hdureader/FITS.java b/src/main/java/fact/io/hdureader/FITS.java index 6b66227be5..304b4e9e53 100644 --- a/src/main/java/fact/io/hdureader/FITS.java +++ b/src/main/java/fact/io/hdureader/FITS.java @@ -102,7 +102,11 @@ public FITS(URL url) throws IOException { } } catch (EOFException e) { - primaryHDU = hdus.get(0); + if (hdus.size() < 1) { + log.warn("File '{}' does not contain any HDUs", url); + } else { + primaryHDU = hdus.get(0); + } stream.close(); log.debug("A total of {} HDUs were found in the file.", hdus.size()); diff --git a/src/main/java/fact/io/hdureader/FITSStream.java b/src/main/java/fact/io/hdureader/FITSStream.java index 56aac8138c..f23c03606c 100644 --- a/src/main/java/fact/io/hdureader/FITSStream.java +++ b/src/main/java/fact/io/hdureader/FITSStream.java @@ -72,7 +72,7 @@ public void init() throws Exception { } // create a refrence to the events hdu - eventHDU = fits.getHDU(nameHDU).orElseThrow(() -> new IOException("Inputfile did not contain HDU '" + nameHDU + "'")); + eventHDU = fits.getHDU(nameHDU).orElseThrow(() -> new MissingHDUException("Inputfile did not contain HDU '" + nameHDU + "'")); BinTable eventsTable = eventHDU.getBinTable(); @@ -137,4 +137,10 @@ public Data readNext() throws Exception { public void skipRows(int amount) throws IOException{ reader.skipRows(amount); } + + public class MissingHDUException extends IOException { + public MissingHDUException(String errorMessage) { + super(errorMessage); + } + } } diff --git a/src/test/java/fact/io/FITSStreamTest.java b/src/test/java/fact/io/FITSStreamTest.java index 39de52a690..800646e365 100644 --- a/src/test/java/fact/io/FITSStreamTest.java +++ b/src/test/java/fact/io/FITSStreamTest.java @@ -61,4 +61,12 @@ public void testFitsKeys() { } } + @Test(expected=FITSStream.MissingHDUException.class) + public void testEmptyFile() throws Exception { + URL u = FITSStreamTest.class.getResource("/empty_fits.fits.gz"); + SourceURL url = new SourceURL(u); + FITSStream stream = new FITSStream(url); + stream.init(); + } + } diff --git a/src/test/resources/empty_fits.fits.gz b/src/test/resources/empty_fits.fits.gz new file mode 100644 index 0000000000000000000000000000000000000000..7f15f29bf8614983a4b3acac003903194aa86c6c GIT binary patch literal 2967 zcmYjScT|(-8n@aYE>MvnLqw5TjDi9(E|nFT7O_xK5|LG)A+i%9K}Z!53$j6mC?gjU zBFg}NAq<&esDP365y$|I5FkJr637ib=iYPwc+Y#zv)Au=)KjFTD^5Ck?Amkg+&QE3 z=g*s*l{|zk!UYu?>2Ko`{s&4zM%YFBhek%64G4;i*i~KV9Wl{#qMPzK2K8k+KeRnB z_tskC;ZJY0HJnSPHH+Xn_CL6^v%ge0m_@V-)7MDtpH(vOfzfZr_0o!;`}L*yVYu_9 zeSh3D7cegl^<)a@qS(=885)e6Y^#Ie8Ll;SpTG^?&B`!cjid?C|5BU5g4$zM@2Jg4$B?Fgc!SzQt}7qv zLbzKt%mPU_hri%Dt3EDU)k96L6Q@n(oG?%0X}XaQ)@o zo2ycwj^vc~dq1=CC>0$g)@df}5i^3PWKjT{Mg!#d8m`QCml7kLfk3r|&c}O)o<=q9fzePvV7zV*r z3+KUIXc%GcP$a-Jwc#8gb8}D!1(SiIb>R`qZk17n!I%eYu`rKR$|&;J>679-@p}j1 zGEvmDSvHY!&*0+x;wYnf)PucPoJXo+kTo5i4Z796|HJP*ec<%>zBNbnyRl%XB^Zo6 z1A@tz%U6AGtNNQ>_OVyb_RRaq^LS^#@Swcdp=)wvwMFO)Y{~I9qPDPdBMZ5U#{4HM zd|9RPaV~642}Oa6mqO$)!>$9^=qLOHe$awlSVlEYI8xSw9tL}aCs*G-Bd{Qrcs#88 z-6Of?8AfZga@5Xe!3##otZKt&r?qDZtqsyv|0B z&Ba|GNND5Gm};VHyiH}W9m-K@?%>eo%9kzxi_ZZ5snDUx&&#Wr{bw$CeY`Y*wM;GG zy$x9btJX|Wzhv?+cdGo3R@Vbi`>le(5(O^aF^w1PYJrHl0Eky%Kfyorh$H0#!?IQf z+KXF5L~{B$SK1~kyYy)uuSugkr_!a^s{4{{S}90yjj zRYu%Ix=sk^bmhyf7KKIaW~1ee*WIuhQ$+eQ%w74oKM2oP>^BLwEN(OK;RBXfaH_~7 zjkn>Ab~^9*PzuE5ldcy-FTK7Y3&df;I&%Z=c=9~H-jG|W)OjhYsdi2ZG{V9wjUhzh zV92ZO@)4=l4^u+|oJ0KpmCEHue7OpCOm=%rz7yb*UEozsBb+e!JHX>wmO4`&%<07< zSaDIz#=a2w7#MnS)yu&R_tX;|m(G9DFJ$+IWYYAOQkN%Kjd!FpRR;DZKR%A0%oa53 z3_pj>yd3rK*bOfLv?M7>Br@hMUSqI;!wAY^t*oF*C;(hPr4+VOz}Q}TTPLinYTMi`Iw?rd7YX^L^aA{&VUn@mOS6ac}kR# z#M4UM%&qR~IpS80I{-Si_EYD1M)DI1DVPv=(!F1sM!trDZ{?tl?iFUe#)d}A9;?4r zTFjj2Hx6s@o_&9ZR8NoU_4JSN0!w7LYPioUQhIQ&6VZ6w9o+5C|KqxYaBZ*KVQCPu;CM=R#WmXgMpi{CKw`h^+}PC2ZHH~eE>vD zwIU!h%Uc+>V>65}%J|3UsdsJ-!tu+9ITq>sh*RgEe~jZl2RCQ)yyhD-%-112vgvbU znb<})wl^L&ls4n+d(>ak%aw5~^^^mlCuIXK(#}fSz(-t(w~)CtF{~v{mJC)=g*nvx zaTx<*S!=O;mV(72{OYM58-QAYNLvDyw^JDI(*)8P9;iXL3qmt9Pe6z++j7(S=(reA zUycan!C8Rk%Gs5}!XMiu;E~vL29Pn}0o7Oc6G#TUPphJ>y)s|tj#uV*Nd1FMclbFm z>5Jqt+vYO!!Yc&UV-Kb)px4{x$| zD>LgT|J|xUM2H~Fu>^X*3kNHHzQy*xCU0DBHAG$i4KuEe6komdH6C__3fw~RevPtH z1a6I3OI9axGdCW7?mee*pDMZ;UM`xou+gA*91x^|gVi-nIDK@LWTjYd%lSY+jCGDv z;lRw>-)08esOyd_CyXz2wqw$K=w@`_Ck7CwM&v<&xFC^hbV0<%H2avgQNwsmt-p6$ zNT&En<38XittM@N@81<>cA6?Z()S8t8HdASy}$P)7S{nGnY)QDaQBwXvw2+eL`n>M zI&>r;{t*{=Ba6y{g8MDE$%kqB{U&;`LLNbev1g4=i)8q6mmZy%mt379C>@!-S-Q5a zrczI{q8V)bnPQ=y^MFmeu;aX;Q5TJAUA@}F(MR7{cYklJAxIbqteZO16LDmDf@y_B zW+U#ee9RUtK8H1SCwUharVl=;m^rMX@aG5IQ|+$Rzgjr$H7>0hCH)Nxc z8)D5EDG+`LzMgm(js2gI@8Z6t>xu*Mbrk+!{MH@v$xvO738-q#-40N@Gs$Rk%l3mD zwR5|D8u4_)46$X6~ zyJw-AHqPC%Le`53;RFVCt!y5t-i@lXn4cCiJ0$03CzrN-zP{mrXNh)Bk|0=I zdO*^}9SoQ+Z8>F47x1E)+h^SJbGH|E Date: Fri, 11 Oct 2019 15:25:07 +0200 Subject: [PATCH 2/4] Add test --- .../java/fact/io/FactFileListMultiStream.java | 21 ++--------- .../java/fact/io/ListMultiStreamTest.java | 36 ++++++++++++++++++- .../dummy_files/file_list_empty.json | 4 +++ 3 files changed, 41 insertions(+), 20 deletions(-) create mode 100644 src/test/resources/dummy_files/file_list_empty.json diff --git a/src/main/java/fact/io/FactFileListMultiStream.java b/src/main/java/fact/io/FactFileListMultiStream.java index 0b983b304f..7f0ada4aa3 100644 --- a/src/main/java/fact/io/FactFileListMultiStream.java +++ b/src/main/java/fact/io/FactFileListMultiStream.java @@ -170,7 +170,7 @@ public Data readNext() throws Exception { } catch (FITSStream.MissingHDUException e) { log.info("File: {} does not contain the Events HDU", stream.getUrl()); if (skipEmpty) { - log.info("Skipping empty file. Continuing with next file."); + log.warn("Skipping empty file {}. Continuing with next file.", stream.getUrl()); stream = null; return this.readNext(); } else { @@ -180,9 +180,8 @@ public Data readNext() throws Exception { } } catch (IOException e) { log.info("File: " + stream.getUrl().toString() + " throws IOException."); - if (skipErrors) { - log.info("Skipping broken files. Continuing with next file."); + log.warn("Skipping broken file {}. Continuing with next file.", stream.getUrl()); e.printStackTrace(); stream = null; return this.readNext(); @@ -205,20 +204,4 @@ public void close() throws Exception { public void setUrl(SourceURL url) { this.url = url; } - - public void setSkipErrors(boolean skipErrors) { - this.skipErrors = skipErrors; - } - - public void setSkipEmpty(boolean skipEmpty) { - this.skipEmpty = skipEmpty; - } - - public void setDrsPathKey(String drsPathKey) { - this.drsPathKey = drsPathKey; - } - - public void setDataPathKey(String dataPathKey) { - this.dataPathKey = dataPathKey; - } } diff --git a/src/test/java/fact/io/ListMultiStreamTest.java b/src/test/java/fact/io/ListMultiStreamTest.java index 88d876ecd9..b9793263e6 100644 --- a/src/test/java/fact/io/ListMultiStreamTest.java +++ b/src/test/java/fact/io/ListMultiStreamTest.java @@ -4,12 +4,13 @@ import org.junit.Test; import stream.Data; import stream.io.SourceURL; +import stream.shell.Run; import java.io.File; import java.net.URL; import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.*; /** * Created by kaibrugge on 14.04.15. @@ -35,12 +36,45 @@ public void testDrsInjection() throws Exception { FITSStream m = new FITSStream(); multiStream.addStream("test", m); + multiStream.skipEmpty = true; multiStream.init(); Data data = multiStream.readNext(); + assertTrue(multiStream.skipEmpty); File drsFile = (File) data.get("@drsFile"); assertThat(drsFile.getName(), is("20140920_66.drs.fits")); } + + @Test + public void testNotSkipEmpty () throws Exception { + URL u = FITSStreamTest.class.getResource("/dummy_files/file_list_empty.json"); + + FactFileListMultiStream multiStream = new FactFileListMultiStream(new SourceURL(u)); + multiStream.setUrl(new SourceURL(u)); + FITSStream m = new FITSStream(); + multiStream.addStream("test", m); + multiStream.init(); + + try { + Data data = multiStream.readNext(); + fail("Runtime exception expected"); + } catch (RuntimeException e) {} + + } + + @Test + public void testSkipEmpty () throws Exception { + URL u = FITSStreamTest.class.getResource("/dummy_files/file_list_empty.json"); + + FactFileListMultiStream multiStream = new FactFileListMultiStream(new SourceURL(u)); + multiStream.skipEmpty = true; + multiStream.setUrl(new SourceURL(u)); + FITSStream m = new FITSStream(); + multiStream.addStream("test", m); + multiStream.init(); + + Data data = multiStream.readNext(); + } } diff --git a/src/test/resources/dummy_files/file_list_empty.json b/src/test/resources/dummy_files/file_list_empty.json new file mode 100644 index 0000000000..1cd82b641c --- /dev/null +++ b/src/test/resources/dummy_files/file_list_empty.json @@ -0,0 +1,4 @@ +[ + {"drs_path":"src\/main\/resources\/dummy_files\/20140920_66.drs.fits","later":"src\/main\/resources\/20140920_66.drs.fits","night":"20140920","data_path":"src\/test\/resources\/empty_fits.fits.gz"}, + {"drs_path":"src\/main\/resources\/dummy_files\/20140920_66.drs.fits","later":"src\/main\/resources\/20140920_66.drs.fits","night":"20140920","data_path":"src\/main\/resources\/testDataFile.fits.gz"} +] \ No newline at end of file From 3ea4fc8e1abd6a405a2904a846d4d9e499ff1a1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20N=C3=B6the?= Date: Fri, 11 Oct 2019 11:31:57 +0200 Subject: [PATCH 3/4] Use openjdk on travis --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8a8e9e1e05..6408621539 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,7 @@ language: java sudo: false script: mvn clean verify jdk: -- oraclejdk8 +- openjdk8 after_script: - bash ./deploy_site.sh @@ -25,7 +25,7 @@ deploy: on: tags: true repo: fact-project/fact-tools - java: oraclejdk8 + java: openjdk8 notifications: slack: From cf4437a243399149d720c8992d3e520dda214e21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20N=C3=B6the?= Date: Fri, 11 Oct 2019 11:32:19 +0200 Subject: [PATCH 4/4] Bump version, add CHANGELOG --- CHANGELOG.md | 5 +++++ pom.xml | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 89ebafa068..804d5e902a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog for the fact-tools +# Version 1.1.2 -- 11.10.2019 + +* Better error message for broken FITS files +* Allow only skipping empty files in FileListMultiStream + # Version 1.1.2 -- 17.12.2018 * Fix reading of uncompressed values from zfits heap tiles (Issue #389) diff --git a/pom.xml b/pom.xml index 086395ec38..ecf9c2959c 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ de.sfb876 fact-tools fact-tools - 1.1.2 + 1.1.3 http://sfb876.de/fact-tools/