diff --git a/pom.xml b/pom.xml
index 5ea790f40c..25e5c3484c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,10 @@
de.sfb876
fact-tools
fact-tools
- 0.18.0
+
+ 0.18.1
+
+
http://sfb876.de/fact-tools/
diff --git a/src/main/java/fact/auxservice/AuxCache.java b/src/main/java/fact/auxservice/AuxCache.java
new file mode 100644
index 0000000000..9b0d3cd929
--- /dev/null
+++ b/src/main/java/fact/auxservice/AuxCache.java
@@ -0,0 +1,91 @@
+package fact.auxservice;
+
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.time.ZonedDateTime;
+
+/**
+ * This class defines some classes needed for caching many {@link AuxPoint} into ram
+ * using an LRU cache. A specific auxfile is uniquely defined by its name and the 'FACT Night' string.
+ * You know...the typical 20160320-like string.
+ *
+ * Created by kai on 12.06.16.
+ */
+
+public class AuxCache {
+ public class CacheKey {
+ final AuxiliaryServiceName service;
+ final Integer factNight;
+ public final Path path;
+ public final String filename;
+
+
+ public CacheKey(AuxiliaryServiceName service, ZonedDateTime timeStamp) {
+ this.service = service;
+ this.factNight = dateTimeStampToFACTNight(timeStamp);
+
+ filename = factNight + "." + service + ".fits";
+ this.path = Paths.get(dateTimeStampToFACTPath(timeStamp).toString(), filename);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ CacheKey cacheKey = (CacheKey) o;
+
+ return service == cacheKey.service && factNight.equals(cacheKey.factNight);
+
+ }
+
+ @Override
+ public int hashCode() {
+ int result = service.hashCode();
+ result = 31 * result + factNight.hashCode();
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ return "CacheKey{" +
+ "service=" + service +
+ ", factNight=" + factNight +
+ '}';
+ }
+ }
+
+ /**
+ * Takes a dateTime object and returns the appropriate FACT night number.
+ * Entering some datetime for 9:30 in the morning, e.g. 2016-01-03 09:30:12, returns 20160102.
+ * The intervall goes to 12:00 noon before its switched to the next night.
+ *
+ * @param timestamp the timestamp to get the night for
+ * @return a FACT night number.
+ */
+ public static Integer dateTimeStampToFACTNight(ZonedDateTime timestamp){
+ ZonedDateTime offsetDate = timestamp.minusHours(12);
+ String night = String.format("%1$d%2$02d%3$02d", offsetDate.getYear(), offsetDate.getMonthValue(), offsetDate.getDayOfMonth());
+ return Integer.parseInt(night);
+ }
+
+
+ /**
+ * Takes a dateTime object and returns the canonical path to an aux or data file.
+ * For example 2016-01-03 09:30:12 returns a path to "2016/01/02" while
+ * 2016-01-03 13:30:12 would return "2016/01/03"
+ *
+ * @param timeStamp the timestamp to get the night for
+ * @return a partial path starting with the year.
+ */
+ public static Path dateTimeStampToFACTPath(ZonedDateTime timeStamp){
+ ZonedDateTime offsetDate = timeStamp.minusHours(12);
+
+ int year = offsetDate.getYear();
+ int month = offsetDate.getMonthValue();
+ int day = offsetDate.getDayOfMonth();
+
+ return Paths.get(String.format("%04d", year), String.format("%02d",month), String.format("%02d", day));
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/fact/auxservice/AuxFileService.java b/src/main/java/fact/auxservice/AuxFileService.java
index f6b98b03d6..f19685fdf2 100644
--- a/src/main/java/fact/auxservice/AuxFileService.java
+++ b/src/main/java/fact/auxservice/AuxFileService.java
@@ -1,47 +1,72 @@
package fact.auxservice;
-//import com.sun.scenario.effect.Offset;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
import fact.auxservice.strategies.AuxPointStrategy;
-
-import fact.io.hdureader.*;
+import fact.io.hdureader.BinTable;
+import fact.io.hdureader.BinTableReader;
+import fact.io.hdureader.FITS;
+import fact.io.hdureader.OptionalTypesMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import stream.annotations.Parameter;
import stream.io.SourceURL;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FilenameFilter;
+import java.io.IOException;
import java.io.Serializable;
-import java.net.MalformedURLException;
-import java.net.URL;
import java.nio.file.Path;
import java.nio.file.Paths;
-import java.time.*;
-import java.util.HashMap;
-import java.util.Map;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import java.util.SortedSet;
import java.util.TreeSet;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
/**
- * This implements an AuxiliaryService {@link fact.auxservice.AuxiliaryService} providing data from the auxiliary
- * files written by the telescopes data acquisition system.
+ * This implements an AuxiliaryService {@link fact.auxservice.AuxiliaryService} providing data from
+ * the auxiliary files written by the telescopes data acquisition system.
+ *
+ * Given the path to the aux folder, that is the folder containing all the auxiliary files for FACT.
+ * This service will read the requested data and store them in a map of {@link fact.auxservice.AuxPoint}.
+ *
+ * Only aux files newer than 2012 are supported!
+ *
+ * Simply provide the 'auxFolder' url to the basepath of the aux files e.g. '/fact/aux/'
+ *
+ * Optionally you can also provide the path to the folder containing the aux data for that specific night e.g. '/fact/aux/2013/01/02/'
*
- * Given the path to the aux folder, that is the folder containing all the auxiliary file for a specific night,
- * via .xml this service will read the requested data and store them in a map of {@link fact.auxservice.AuxPoint}.
+ * Because one might want read from multiple sources at once, and many streams are accessing this service at once, or
+ * one simply wants to access many different aux files the data from one file is cached into a guava cache.
+ * This saves us the overhead of keeping tracks of different files in some custom structure.
*
* Created by kaibrugge on 07.10.14.
*/
public class AuxFileService implements AuxiliaryService {
- Logger log = LoggerFactory.getLogger(AuxFileService.class);
+ private Logger log = LoggerFactory.getLogger(AuxFileService.class);
- Map> services = new HashMap<>();
+ @Parameter(required = true, description = "The url pointing to the path containing a the auxilary " +
+ "data in FACTS canonical folder structure." )
+ public SourceURL auxFolder;
+ public void setAuxFolder(SourceURL auxFolder) {
+ this.auxFolder = auxFolder;
+ }
- @Parameter(required = false, description = "The path to the folder containing the auxilary data as .fits files")
- SourceURL auxFolder;
- boolean isInit = false;
- private HashMap auxFileUrls;
+ private LoadingCache> cache = CacheBuilder.newBuilder()
+ .maximumSize(100)
+ .expireAfterAccess(20, TimeUnit.MINUTES)
+ .removalListener(notification -> log.debug("Removing Data from cache for cause {}", notification.getCause()))
+ .build(new CacheLoader>() {
+ @Override
+ public TreeSet load(AuxCache.CacheKey key) throws Exception {
+ return readDataFromFile(key);
+ }
+ });
+
/**
* This method returns an AuxPoint according to the strategy and the time stamp passed to this method.
@@ -53,123 +78,96 @@ public class AuxFileService implements AuxiliaryService {
* @param serviceName The name of the service.
* @param eventTimeStamp The time stamp of the current raw data event.
* @param strategy One of the strategies provided.
- * @return
- * @throws FileNotFoundException
+ * @return the auxpoint selected by the strategy if it exists.
+ * @throws IOException when no auxpoint can be found for given timestamp
*/
@Override
- public AuxPoint getAuxiliaryData(AuxiliaryServiceName serviceName, ZonedDateTime eventTimeStamp, AuxPointStrategy strategy) throws FileNotFoundException {
- if(!isInit){
- auxFileUrls = findAuxFileUrls(auxFolder);
- isInit = true;
+ public synchronized AuxPoint getAuxiliaryData(AuxiliaryServiceName serviceName, ZonedDateTime eventTimeStamp, AuxPointStrategy strategy) throws IOException {
+ if(eventTimeStamp.isAfter(ZonedDateTime.now())){
+ log.warn("The requested timestamp seems to be in the future.");
}
- if(!services.containsKey(serviceName)){
- services.put(serviceName, readDataFromFile(auxFileUrls.get(serviceName), serviceName.toString()));
- }
- TreeSet set = services.get(serviceName);
+ try {
+ AuxCache.CacheKey key = new AuxCache().new CacheKey(serviceName, eventTimeStamp);
- ZonedDateTime firstTimeStamp = set.first().getTimeStamp();
- ZonedDateTime lastTimeStamp = set.last().getTimeStamp();
- if(firstTimeStamp.isAfter(eventTimeStamp) || lastTimeStamp.isBefore(eventTimeStamp))
- {
- log.warn("Provided event timestamp not in auxiliary File.");
- }
+ TreeSet auxPoints = cache.get(key);
+ AuxPoint pointFromTreeSet = strategy.getPointFromTreeSet(auxPoints, eventTimeStamp);
+ if (pointFromTreeSet == null) {
+ throw new IOException("No auxpoint found for the given timestamp " + eventTimeStamp);
+ }
+ return pointFromTreeSet;
- //TODO: load a new file in case we need stuff from the next day or night. I don't know whether this is ever a valid use case.
- return strategy.getPointFromTreeSet(set, eventTimeStamp);
+ } catch (ExecutionException e) {
+ throw new IOException("No auxpoint found for the given timestamp " + eventTimeStamp);
+ }
}
/**
- * Reads data from a file provided by the url and creates an AuxPoint for each event in the file.
- * @param auxFileUrl url to the auxfile
- * @return treeset containing auxpoints ordered by their timestamp
+ * This method returns all AuxPoints for the whole night given by the 'night' timestamp.
+ *
+ * @throws IOException when no auxpoint can be found for the given night
*/
- private TreeSet readDataFromFile(SourceURL auxFileUrl, String extname) {
- TreeSet result = new TreeSet<>();
-
- //create a fits object
+ public synchronized SortedSet getAuxiliaryDataForWholeNight(AuxiliaryServiceName serviceName, ZonedDateTime night) throws IOException {
try {
- URL url = new URL(auxFileUrl.getProtocol(), auxFileUrl.getHost(), auxFileUrl.getPort(), auxFileUrl.getFile());
- FITS fits = new FITS(url);
- BinTable auxDataBinTable = fits.getBinTableByName(extname)
- .orElseThrow(
- () -> new RuntimeException("BinTable '" + extname + "' not in aux file")
- );
- BinTableReader auxDataBinTableReader = BinTableReader.forBinTable(auxDataBinTable);
-
- while (auxDataBinTableReader.hasNext()) {
- OptionalTypesMap auxData = auxDataBinTableReader.getNextRow();
-
- auxData.getDouble("Time").ifPresent(time -> {
- long value = (long) (time * 24 * 60 * 60 *1000 );
- ZonedDateTime t = Instant.ofEpochMilli(value).atZone(ZoneOffset.UTC);
- AuxPoint p = new AuxPoint(t, auxData);
- result.add(p);
- });
+ AuxCache.CacheKey key = new AuxCache().new CacheKey(serviceName, night);
+
+ TreeSet auxPoints = cache.get(key);
+ if (auxPoints.isEmpty()){
+ throw new IOException("No auxpoints found for the given night " + night);
}
- return result;
- } catch (Exception e) {
- log.error("Failed to load data from AUX file: {}", e.getMessage());
- throw new RuntimeException(e);
+ return auxPoints;
+
+ } catch (ExecutionException e) {
+ throw new IOException("No auxpoints found for the given night" + night);
}
}
+ private TreeSet readDataFromFile(AuxCache.CacheKey key) throws Exception {
+ TreeSet result = new TreeSet<>();
- /**
- * Finds all .fits file in the given folder that contain one of the values from AuxiliaryServiceName in their
- * file name. This is public for unit testing purposes.
- * @param auxFolder
- * @return a mapping from a AuxiliaryServiceName to a SourceURL which points to a file.
- * @throws java.io.FileNotFoundException in case the provided URL doesnt point to a readable folder.
- */
- public HashMap findAuxFileUrls(SourceURL auxFolder) throws FileNotFoundException {
+ Path pathToFile = Paths.get(auxFolder.getPath(), key.path.toString());
+
+ if(pathToFile == null){
+ log.error("Could not load aux file {} for night {}", key.service, key.factNight);
+ throw new IOException("Could not load aux file for key " + key);
+ }
- Path p = Paths.get(auxFolder.getPath());
- File folder = p.toFile();
+ //test whether file is in current directory. this ensures compatibility to fact-tools version < 18.0
+ if (!pathToFile.toFile().canRead()){
+ pathToFile = Paths.get(auxFolder.getPath(), key.filename);
- if(!folder.exists()){
- throw new FileNotFoundException("The path does not exist: " + folder.toString());
+ if (!pathToFile.toFile().canRead()){
+ log.error("Could not load aux file in given directory {}", auxFolder);
+ }
}
- if(!folder.isDirectory()){
- throw new FileNotFoundException("The path does not point to a directory: " + folder.toString());
+
+
+ FITS fits = FITS.fromPath(pathToFile);
+ String extName = key.service.name();
+
+ BinTable auxDataBinTable = fits.getBinTableByName(extName)
+ .orElseThrow(
+ () -> new RuntimeException("BinTable '" + extName + "' not in aux file")
+ );
+ BinTableReader auxDataBinTableReader = BinTableReader.forBinTable(auxDataBinTable);
+
+ while (auxDataBinTableReader.hasNext()) {
+ OptionalTypesMap auxData = auxDataBinTableReader.getNextRow();
+
+ auxData.getDouble("Time").ifPresent(time -> {
+ long value = (long) (time * 24 * 60 * 60 *1000 );
+ ZonedDateTime t = Instant.ofEpochMilli(value).atZone(ZoneOffset.UTC);
+ AuxPoint p = new AuxPoint(t, auxData);
+ result.add(p);
+ });
+
}
- final HashMap m = new HashMap<>();
- folder.list(new FilenameFilter() {
- @Override
- public boolean accept(File dir, String name) {
- if (name.endsWith(".fits")) {
- try {
- //get name of aux file by removing the date string (first 9 characters) and the file ending
- String auxName = name.substring(9);
- auxName = auxName.substring(0, auxName.length() - 5);
- File f = new File(dir, name);
- m.put(AuxiliaryServiceName.valueOf(auxName), new SourceURL(f.toURI().toURL()));
- } catch (MalformedURLException e) {
- log.error("Could not create path to auxillary file " + dir + " " +name);
- return false;
- }catch (IllegalArgumentException e) {
- log.warn("The file " + dir + "/" +name + " is not a recognized aux service. ");
- return false;
- }catch (IndexOutOfBoundsException e){
- log.warn("The file " + dir + "/" +name + " is not a recognized aux service. " +
- "Could not parse file name into a recognized service.");
- return false;
- }
- return true;
- }
- return false;
- }
- });
- return m;
+ return result;
}
+
@Override
public void reset() throws Exception {
}
- public void setAuxFolder(SourceURL auxFolder) {
- this.auxFolder = auxFolder;
- }
-
-
}
diff --git a/src/main/java/fact/auxservice/AuxPoint.java b/src/main/java/fact/auxservice/AuxPoint.java
index c53f1f296f..c6d7b07f3d 100644
--- a/src/main/java/fact/auxservice/AuxPoint.java
+++ b/src/main/java/fact/auxservice/AuxPoint.java
@@ -6,6 +6,7 @@
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
import java.util.Map;
+import java.util.Optional;
/**
* This is a class to create immutable container holding data from some source providing auxiliary data.
@@ -47,7 +48,8 @@ public ImmutableMap getData() {
/**
* Returns the value for the key iff it exists and its a Double. Returns null otherwise.
- * @param key
+ *
+ * @param key the name of the aux data you want to access. e.g. the name of the column in the aux fits file
* @return the value or null
*/
public Double getDouble(String key){
@@ -58,9 +60,21 @@ public Double getDouble(String key){
}
}
+ /**
+ * Returns the value for the key iff it exists and its a Float. Returns null otherwise.
+ * @param key the name of the aux data you want to access. e.g. the name of the column in the aux fits file
+ * @return the value or null
+ */
+ public Float getFloat(String key){
+ try {
+ return (Float) data.get(key);
+ } catch (ClassCastException e){
+ return null;
+ }
+ }
/**
* Returns the value for the key iff it exists and its an Integer. Returns null otherwise.
- * @param key
+ * @param key the name of the aux data you want to access. e.g. the name of the column in the aux fits file
* @return the value or null
*/
public Integer getInteger(String key){
@@ -71,6 +85,11 @@ public Integer getInteger(String key){
}
}
+ /**
+ * Returns the value for the key iff it exists and its a String. Returns null otherwise.
+ * @param key the name of the aux data you want to access. e.g. the name of the column in the aux fits file
+ * @return the value or null
+ */
public String getString(String key){
try {
return (String) data.get(key);
@@ -81,7 +100,7 @@ public String getString(String key){
/**
* Returns the value for the key iff it exists and its an int[]. Returns null otherwise.
- * @param key
+ * @param key the name of the aux data you want to access. e.g. the name of the column in the aux fits file
* @return the value or null
*/
public int[] getIntegerArray(String key){
@@ -94,7 +113,7 @@ public int[] getIntegerArray(String key){
/**
* Returns the value for the key iff it exists and its an int[]. Returns null otherwise.
- * @param key
+ * @param key the name of the aux data you want to access. e.g. the name of the column in the aux fits file
* @return the value or null
*/
public double[] getDoubleArray(String key){
@@ -106,6 +125,24 @@ public double[] getDoubleArray(String key){
}
+ /**
+ * Returns an Optional containing the value with the given class iff it exists. Otherwise its empty.
+ * @param key the name of the aux data you want to access. e.g. the name of the column in the aux fits file
+ * @param cls the data type you expect the value to be in
+ * @return an optional containing the value or an empty optional.
+ */
+ public Optional getValue(String key, Class cls){
+ Serializable s = data.get(key);
+ if (s == null){
+ return Optional.empty();
+ }
+ if (cls.isInstance(s)){
+ return Optional.of(cls.cast(s));
+ } else{
+ return Optional.empty();
+ }
+ }
+
//below you'll find the auto generated code needed to make this object hashable and comparable
@Override
diff --git a/src/main/java/fact/auxservice/AuxWebService.java b/src/main/java/fact/auxservice/AuxWebService.java
index 54722ab4c8..c4d8a60690 100644
--- a/src/main/java/fact/auxservice/AuxWebService.java
+++ b/src/main/java/fact/auxservice/AuxWebService.java
@@ -13,7 +13,9 @@
import java.io.IOException;
import java.io.Serializable;
import java.lang.reflect.Type;
-import java.time.*;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
import java.util.HashMap;
import java.util.Map;
import java.util.TreeSet;
diff --git a/src/main/java/fact/auxservice/AuxiliaryService.java b/src/main/java/fact/auxservice/AuxiliaryService.java
index 53bcd2f01c..9c3445d55e 100644
--- a/src/main/java/fact/auxservice/AuxiliaryService.java
+++ b/src/main/java/fact/auxservice/AuxiliaryService.java
@@ -1,10 +1,15 @@
package fact.auxservice;
import fact.auxservice.strategies.AuxPointStrategy;
+import stream.Data;
import stream.service.Service;
import java.io.IOException;
+import java.time.Instant;
+import java.time.ZoneOffset;
import java.time.ZonedDateTime;
+import java.util.Optional;
+
/**
* The service should provide the ability to get the aux data from some data source.
@@ -19,11 +24,39 @@ public interface AuxiliaryService extends Service {
* Providing the timestamp of the event, the name of the service
* and some strategy, this method should return an AuxPoint.
*
- * @param serviceName
- * @param eventTimeStamp
- * @param strategy
- * @return
- * @throws IOException
+ * @param serviceName the name of the aux data to access. This is written in the filename 20130112..fits
+ * @param eventTimeStamp the DateTime of the event you need the aux data for.
+ * @param strategy one of the strategies implemented for fetcvhing aux points
+ * @return the auxpoint according to the strategy
+ * @throws IOException in case something goes wrong while trying to access the aux data. Be it a file or a database.
+ */
+ AuxPoint getAuxiliaryData(AuxiliaryServiceName serviceName, ZonedDateTime eventTimeStamp, AuxPointStrategy strategy) throws IOException;
+
+ /**
+ * Takes the int[2] array found in the FITs files under the name UnixTimeUTC and converts it to a DateTime
+ * instance with time zone UTC. If the passed array cannot be converted the optional will be empty.
+ *
+ * @param eventTime the UnixTimeUTC array as found in the FITS file.
+ * @return an Optional containing the Datetime instance
+ */
+ static Optional unixTimeUTCToDateTime(int [] eventTime){
+ if(eventTime != null && eventTime.length == 2) {
+ long value = (long)((eventTime[0]+eventTime[1]/1000000.)*1000);
+ ZonedDateTime timeStamp = Instant.ofEpochMilli(value).atZone(ZoneOffset.UTC);
+ return Optional.of(timeStamp);
+ }
+ return Optional.empty();
+ }
+
+ /**
+ * Takes the int[2] array found in the FITs files under the name UnixTimeUTC from the Data Item and converts it to a DateTime
+ * instance with time zone UTC. If the passed array cannot be converted the optional will be empty.
+ *
+ * @param item A data item from the stream of raw FACT data
+ * @return an Optional containing the Datetime instance
*/
- public AuxPoint getAuxiliaryData(AuxiliaryServiceName serviceName, ZonedDateTime eventTimeStamp, AuxPointStrategy strategy) throws IOException;
+ static Optional unixTimeUTCToDateTime(Data item){
+ int[] eventTime = (int[]) item.get("UnixTimeUTC");
+ return unixTimeUTCToDateTime(eventTime);
+ }
}
diff --git a/src/main/java/fact/auxservice/SqliteService.java b/src/main/java/fact/auxservice/SqliteService.java
index e9fccbb5dc..15ee0785fe 100644
--- a/src/main/java/fact/auxservice/SqliteService.java
+++ b/src/main/java/fact/auxservice/SqliteService.java
@@ -6,14 +6,17 @@
import org.slf4j.LoggerFactory;
import org.tmatesoft.sqljet.core.SqlJetException;
import org.tmatesoft.sqljet.core.SqlJetTransactionMode;
-import org.tmatesoft.sqljet.core.table.*;
+import org.tmatesoft.sqljet.core.table.ISqlJetCursor;
+import org.tmatesoft.sqljet.core.table.ISqlJetTable;
+import org.tmatesoft.sqljet.core.table.SqlJetDb;
import stream.annotations.Parameter;
import stream.io.SourceURL;
import java.io.File;
import java.io.IOException;
import java.io.Serializable;
-import java.time.*;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
import java.util.HashMap;
import java.util.Map;
import java.util.TreeSet;
diff --git a/src/main/java/fact/auxservice/strategies/AuxPointStrategy.java b/src/main/java/fact/auxservice/strategies/AuxPointStrategy.java
index f874f0b106..36b6b39e08 100644
--- a/src/main/java/fact/auxservice/strategies/AuxPointStrategy.java
+++ b/src/main/java/fact/auxservice/strategies/AuxPointStrategy.java
@@ -2,15 +2,17 @@
import fact.auxservice.AuxPoint;
-import java.time.ZonedDateTime;
import java.time.ZonedDateTime;
import java.util.TreeSet;
/**
* An AuxPointStrategy defines which point from a given TreeSet should be returned.
* You could also interpolate between two results and return a new interpolated Point.
- * This can return null if the data doesnt exist in this context. For example if the File from which the data
- * was read does not contain a valid point for that eventTimeStamp
+ * This can return null if the data doesn't exist in this context. For example if the File from which the data
+ * was read does not contain a valid point for that eventTimeStamp.
+ *
+ * This interface also defines convenience functions for creating these strategy objects.
+ *
* Created by kai on 01.04.15.
*/
public interface AuxPointStrategy {
@@ -20,5 +22,31 @@ public interface AuxPointStrategy {
* @param eventTimeStamp the timestamp for which you want the auxiliary data
* @return an AuxPoint according to the concrete strategy implementation.
*/
- public AuxPoint getPointFromTreeSet(TreeSet set, ZonedDateTime eventTimeStamp);
+ AuxPoint getPointFromTreeSet(TreeSet set, ZonedDateTime eventTimeStamp);
+
+ /**
+ * Convenience method to create Strategy object
+ * @return strategy to get the closest AuxPoint in Time.
+ */
+ static AuxPointStrategy Closest(){
+ return new Closest();
+ }
+
+
+ /**
+ * Convenience method to create Strategy object
+ * @return strategy to get the closest AuxPoint that comes earlier in Time.
+ */
+ static AuxPointStrategy Earlier(){
+ return new Earlier();
+ }
+
+
+ /**
+ * Convenience method to create Strategy object
+ * @return strategy to get the closest AuxPoint that comes later in Time.
+ */
+ static AuxPointStrategy Later(){
+ return new Later();
+ }
}
diff --git a/src/main/java/fact/auxservice/strategies/Closest.java b/src/main/java/fact/auxservice/strategies/Closest.java
index 2dba47b252..7b2a8e55c0 100644
--- a/src/main/java/fact/auxservice/strategies/Closest.java
+++ b/src/main/java/fact/auxservice/strategies/Closest.java
@@ -4,7 +4,6 @@
import java.time.Duration;
import java.time.ZonedDateTime;
-import java.time.ZonedDateTime;
import java.util.TreeSet;
/**
diff --git a/src/main/resources/aux/20130102.DRIVE_CONTROL_SOURCE_POSITION.fits b/src/main/resources/aux/2013/01/02/20130102.DRIVE_CONTROL_SOURCE_POSITION.fits
similarity index 100%
rename from src/main/resources/aux/20130102.DRIVE_CONTROL_SOURCE_POSITION.fits
rename to src/main/resources/aux/2013/01/02/20130102.DRIVE_CONTROL_SOURCE_POSITION.fits
diff --git a/src/main/resources/aux/20130102.DRIVE_CONTROL_TRACKING_POSITION.fits b/src/main/resources/aux/2013/01/02/20130102.DRIVE_CONTROL_TRACKING_POSITION.fits
similarity index 100%
rename from src/main/resources/aux/20130102.DRIVE_CONTROL_TRACKING_POSITION.fits
rename to src/main/resources/aux/2013/01/02/20130102.DRIVE_CONTROL_TRACKING_POSITION.fits
diff --git a/src/test/java/fact/services/AuxServiceTest.java b/src/test/java/fact/services/AuxServiceTest.java
index e57bde7e0d..66fd1cd7b3 100644
--- a/src/test/java/fact/services/AuxServiceTest.java
+++ b/src/test/java/fact/services/AuxServiceTest.java
@@ -1,48 +1,176 @@
package fact.services;
+
+import fact.auxservice.AuxCache;
+import fact.auxservice.AuxPoint;
import fact.auxservice.AuxiliaryServiceName;
import fact.auxservice.AuxFileService;
+import fact.auxservice.strategies.Closest;
+import org.joda.time.DateTime;
import org.junit.Test;
import stream.io.SourceURL;
-import java.io.FileNotFoundException;
import java.net.URL;
-import java.util.HashMap;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.time.ZonedDateTime;
+import java.util.Optional;
+import java.util.SortedSet;
+
+import static org.hamcrest.core.Is.is;
+import static org.hamcrest.core.IsNot.not;
+import static org.hamcrest.core.IsNull.nullValue;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
+
/**
+ * Test some things about the AuxFileService
* Created by kaibrugge on 17.11.14.
*/
public class AuxServiceTest {
- /**
- * Tests what happens when the folder to the datestring exists but the files in that folder
- * don't have the right datestring. We expect an empty map in response.
- * @throws Exception
- */
@Test
- public void testWrongFilename() throws Exception {
- URL u = AuxServiceTest.class.getResource("/dummy_files/aux/2015/09/");
- AuxFileService s = new AuxFileService();
- //supply a wrong datestring
-// String dateString = "20150921";
- HashMap m = s.findAuxFileUrls(new SourceURL(u));
- assertTrue(m.isEmpty());
+ public void testDateTimeToFACTNight(){
+ Integer factNight = AuxCache.dateTimeStampToFACTNight(ZonedDateTime.parse("2014-01-02T23:55:02Z"));
+ assertThat(factNight, is(20140102));
+
+ //now test what happens after 00:00
+ factNight = AuxCache.dateTimeStampToFACTNight(ZonedDateTime.parse("2014-01-03T00:55:02Z"));
+ assertThat(factNight, is(20140102));
+
+ //now test the next day
+ factNight = AuxCache.dateTimeStampToFACTNight(ZonedDateTime.parse("2014-01-03T12:00:01Z"));
+ assertThat(factNight, is(20140103));
+
+ //now test what happens after newyears
+ factNight = AuxCache.dateTimeStampToFACTNight(ZonedDateTime.parse("2014-01-01T00:55:02Z"));
+ assertThat(factNight, is(20131231));
+
+ //now test my birfday!!
+ factNight = AuxCache.dateTimeStampToFACTNight(ZonedDateTime.parse("1987-09-20T20:55:02Z"));
+ assertThat(factNight, is(19870920));
+ }
+
+
+ @Test
+ public void testDateTimeToFACTPath(){
+ Path factNight = AuxCache.dateTimeStampToFACTPath(ZonedDateTime.parse("2014-01-02T23:55:02Z"));
+
+ assertThat(factNight, is(Paths.get("2014", "01", "02")));
+
+ //now test what happens after 00:00
+ factNight = AuxCache.dateTimeStampToFACTPath(ZonedDateTime.parse("2014-01-03T00:55:02Z"));
+ assertThat(factNight, is(Paths.get("2014", "01", "02")));
+
+ //now test the next day
+ factNight = AuxCache.dateTimeStampToFACTPath(ZonedDateTime.parse("2014-01-03T12:00:01Z"));
+ assertThat(factNight, is(Paths.get("2014", "01", "03")));
+
+ //now test what happens after newyears
+ factNight = AuxCache.dateTimeStampToFACTPath(ZonedDateTime.parse("2014-01-01T00:55:02Z"));
+ assertThat(factNight, is(Paths.get("2013", "12", "31")));
+
+ //now test my birfday!!
+ factNight = AuxCache.dateTimeStampToFACTPath(ZonedDateTime.parse("1987-09-20T20:55:02Z"));
+ assertThat(factNight, is(Paths.get("1987", "09", "20")));
}
@Test
public void testAuxFileFinder() throws Exception {
- URL u = AuxServiceTest.class.getResource("/dummy_files/aux/2015/09/21");
+ URL u = AuxServiceTest.class.getResource("/dummy_files/aux/");
// SourceURL url = new SourceURL(u);
+ AuxFileService auxFileService = new AuxFileService();
+ auxFileService.auxFolder = new SourceURL(u);
+
+ ZonedDateTime night = ZonedDateTime.parse("2016-09-20T20:55:02Z");
+ AuxCache.CacheKey key = new AuxCache().new CacheKey(AuxiliaryServiceName.FTM_CONTROL_STATE, night);
+ Path path = Paths.get(auxFileService.auxFolder.getPath(), key.path.toString());
+ //file should not exist
+ assertFalse(path.toFile().exists());
+
+
+
+ night = ZonedDateTime.parse("2014-09-20T20:55:02Z");
+ key = new AuxCache().new CacheKey(AuxiliaryServiceName.FTM_CONTROL_STATE, night);
+ path = Paths.get(auxFileService.auxFolder.getPath(), key.path.toString());
+
+ assertTrue(path.toFile().exists());
+
+
+ night = ZonedDateTime.parse("2014-09-20T20:55:02Z");
+ key = new AuxCache().new CacheKey(AuxiliaryServiceName.RATE_SCAN_PROCESS_DATA, night);
+ path = Paths.get(auxFileService.auxFolder.getPath(), key.path.toString());
+
+ assertTrue(path.toFile().exists());
+
+
+
+ night = ZonedDateTime.parse("2013-01-02T20:55:02Z");
+ key = new AuxCache().new CacheKey(AuxiliaryServiceName.DRIVE_CONTROL_TRACKING_POSITION, night);
+ path = Paths.get(auxFileService.auxFolder.getPath(), key.path.toString());
+
+ assertTrue(path.toFile().exists());
+
+
+
+ night = ZonedDateTime.parse("2014-09-20T20:55:02Z");
+ key = new AuxCache().new CacheKey(AuxiliaryServiceName.DRIVE_CONTROL_TRACKING_POSITION, night);
+ path = Paths.get(auxFileService.auxFolder.getPath(), key.path.toString());
+ //file should not exist
+ assertFalse(path.toFile().exists());
+ }
+
+ @Test
+ public void testAuxFileService() throws Exception {
+ URL u = AuxServiceTest.class.getResource("/dummy_files/aux/");
AuxFileService s = new AuxFileService();
- //supply datestring. ITS MY BIRFDAY! YAY
-// String dateString = "20150920";
- HashMap m = s.findAuxFileUrls(new SourceURL(u));
- assertTrue(m.containsKey(AuxiliaryServiceName.DRIVE_CONTROL_TRACKING_POSITION));
-// assertTrue(m.containsKey("DRIVE_CONTROL_POINTING_POSITION"));
+ s.auxFolder = new SourceURL(u);
+ AuxPoint auxiliaryData = s.getAuxiliaryData(AuxiliaryServiceName.DRIVE_CONTROL_TRACKING_POSITION, ZonedDateTime.parse("2013-01-02T23:30:21Z"), new Closest());
+ assertThat(auxiliaryData, is(not(nullValue())));
+ assertThat(auxiliaryData.getDouble("Ra"), is(not(nullValue())));
+ }
+
+
+
+ @Test
+ public void testAuxPointGenericValue() throws Exception {
+ URL u = AuxServiceTest.class.getResource("/dummy_files/aux/");
+ AuxFileService s = new AuxFileService();
+ s.auxFolder = new SourceURL(u);
+ AuxPoint point = s.getAuxiliaryData(AuxiliaryServiceName.DRIVE_CONTROL_TRACKING_POSITION, ZonedDateTime.parse("2013-01-02T21:30:21Z"), new Closest());
+ Optional ra = point.getValue("Ra", Double.class);
+ ra.orElseThrow(() -> new RuntimeException("Value is missing"));
+ }
+
+ /**
+ * Test whether files in the specific aux folder are found
+ */
+ @Test
+ public void testAuxFileServiceForSpecificLocation() throws Exception {
+ URL u = AuxServiceTest.class.getResource("/dummy_files/aux/2013/01/02");
+ AuxFileService s = new AuxFileService();
+ s.auxFolder = new SourceURL(u);
+ AuxPoint auxiliaryData = s.getAuxiliaryData(AuxiliaryServiceName.DRIVE_CONTROL_TRACKING_POSITION, ZonedDateTime.parse("2013-01-02T23:30:21Z"), new Closest());
+ assertThat(auxiliaryData, is(not(nullValue())));
+ assertThat(auxiliaryData.getDouble("Ra"), is(not(nullValue())));
+ }
+
+ @Test
+ public void testToGetAllPointsForNight() throws Exception {
+ URL u = AuxServiceTest.class.getResource("/dummy_files/aux/");
+ AuxFileService s = new AuxFileService();
+ s.auxFolder = new SourceURL(u);
+ SortedSet auxiliaryData = s.getAuxiliaryDataForWholeNight(
+ AuxiliaryServiceName.DRIVE_CONTROL_TRACKING_POSITION,
+ ZonedDateTime.parse("2013-01-02T23:30:21Z")
+ );
+
+ assertFalse(auxiliaryData.isEmpty());
}
}
diff --git a/src/test/resources/dummy_files/aux/2013/01/02/20130102.DRIVE_CONTROL_TRACKING_POSITION.fits b/src/test/resources/dummy_files/aux/2013/01/02/20130102.DRIVE_CONTROL_TRACKING_POSITION.fits
new file mode 100644
index 0000000000..dda614aa1d
--- /dev/null
+++ b/src/test/resources/dummy_files/aux/2013/01/02/20130102.DRIVE_CONTROL_TRACKING_POSITION.fits
@@ -0,0 +1,2923 @@
+SIMPLE = T / file does conform to FITS standard BITPIX = 8 / number of bits per data pixel NAXIS = 0 / number of data axes EXTEND = T / FITS dataset may contain extensions COMMENT FITS (Flexible Image Transport System) format is defined in 'AstronomyCOMMENT and Astrophysics', volume 376, page 359; bibcode: 2001A&A...376..359H END XTENSION= 'BINTABLE' / binary table extension BITPIX = 8 / 8-bit bytes NAXIS = 2 / 2-dimensional binary table NAXIS1 = 76 / width of table in bytes NAXIS2 = 12113 / number of rows in table PCOUNT = 0 / size of special data area GCOUNT = 1 / one data group (required keyword) TFIELDS = 10 / number of fields in each row TTYPE1 = 'Time ' / Modified Julian Date TFORM1 = '1D ' / data format of field: 8-byte DOUBLE TUNIT1 = 'MJD ' / physical unit of field TTYPE2 = 'QoS ' / Quality of service TFORM2 = '1J ' / data format of field: 4-byte INTEGER TTYPE3 = 'Ra ' / Command right ascension TFORM3 = '1D ' / data format of field: 8-byte DOUBLE TUNIT3 = 'h ' / physical unit of field TTYPE4 = 'Dec ' / Command declination TFORM4 = '1D ' / data format of field: 8-byte DOUBLE TUNIT4 = 'deg ' / physical unit of field TTYPE5 = 'Ha ' / Corresponding hour angle TFORM5 = '1D ' / data format of field: 8-byte DOUBLE TUNIT5 = 'h ' / physical unit of field TTYPE6 = 'Zd ' / Nominal zenith distance TFORM6 = '1D ' / data format of field: 8-byte DOUBLE TUNIT6 = 'deg ' / physical unit of field TTYPE7 = 'Az ' / Nominal azimuth angle TFORM7 = '1D ' / data format of field: 8-byte DOUBLE TUNIT7 = 'deg ' / physical unit of field TTYPE8 = 'dZd ' / Control deviation Zd TFORM8 = '1D ' / data format of field: 8-byte DOUBLE TUNIT8 = 'deg ' / physical unit of field TTYPE9 = 'dAz ' / Control deviation Az TFORM9 = '1D ' / data format of field: 8-byte DOUBLE TUNIT9 = 'deg ' / physical unit of field TTYPE10 = 'dev ' / Absolute control deviation TFORM10 = '1D ' / data format of field: 8-byte DOUBLE TUNIT10 = 'arcsec ' / physical unit of field EXTNAME = 'DRIVE_CONTROL_TRACKING_POSITION' / name of this binary table extensioTCOMM1 = 'Modified Julian Date' TCOMM2 = 'Quality of service' TCOMM3 = 'Command right ascension' TCOMM4 = 'Command declination' TCOMM5 = 'Corresponding hour angle' TCOMM6 = 'Nominal zenith distance' TCOMM7 = 'Nominal azimuth angle' TCOMM8 = 'Control deviation Zd' TCOMM9 = 'Control deviation Az' TCOMM10 = 'Absolute control deviation' COMMENT = ' ' TELESCOP= 'FACT ' / Telescope that acquired this data PACKAGE = 'FACT++ ' / Package name VERSION = '1.0 ' / Package description CREATOR = 'datalogger' / Program that wrote this file EXTREL = 1. / Release Number COMPILED= 'Dec 3 2012 14:48:01' / Compile time REVISION= '14491:14715M' / SVN revision ORIGIN = 'FACT ' / Institution that wrote the file DATE = '2013-01-02T20:08:05.177080' / File creation date NIGHT = 20130102 / Night as int TIMESYS = 'UTC ' / Time system TIMEUNIT= 'd ' / Time given in days w.r.t. to MJDREF MJDREF = 40587 / Store times in UNIX time (for convenience, secoTSTARTI = 15707 / Time when first event received (integral part) TSTARTF = 0.838946990741533 / Time when first event received (fractional partTSTOPI = 15708 / Time when last event received (integral part) TSTOPF = 0.196680000000924 / Time when last event received (fractional part)DATE-OBS= '2013-01-02T20:08:05.019999' / Time when first event received DATE-END= '2013-01-03T04:43:13.152000' / Time when last event received RUNID = 0 / Run number. 0 means not run file END @έbp( @o7HZ@6fAu1 @IP?/qO4VM`?4#5@ p=@έc @o7HZ@6fAu133333@I?.qO,_ф&m?Rԉr@ݕ3