Timeseries sorting is now done on numeric keys is possible

CCDFDataModel/CCDFTypes.cpp

@@ -241,3 +241,13 @@ int CDF::fill(void *destdata, CDFType destType, double value, size_t size) {
   }
   return 0;
 }
+
+bool CDF::isCDFNumeric(CDFType type) {
+  switch (type) {
+  case CDF_CHAR:
+  case CDF_STRING:
+    return false;
+  default:
+    return true;
+  }
+}

CCDFDataModel/CCDFTypes.h

@@ -34,8 +34,8 @@
 
 #include "CTypes.h"
 
-//#define CCDFDATAMODEL_DEBUG
-// CDF: Common Data Format
+// #define CCDFDATAMODEL_DEBUG
+//  CDF: Common Data Format
 
 /* Types supported by CDF */
 typedef int CDFType;
@@ -76,6 +76,8 @@ namespace CDF {
   int allocateData(CDFType type, void **p, size_t length);
   int freeData(void **p);
 
+  // Check if this is a string type or a numeric type
+  bool isCDFNumeric(CDFType type);
   // Copies data from one array to another and performs type conversion
   // Destdata must be a pointer to an empty array with non-void type
   class DataCopier {

CCDFDataModel/CCDFVariable.h

@@ -32,7 +32,10 @@
 #include "CCDFDimension.h"
 // #define CCDFDATAMODEL_DEBUG
 #include "CDebugger.h"
+
+class CDFObject;
 namespace CDF {
+
   class Variable {
 
   public:
@@ -58,7 +61,7 @@ namespace CDF {
   private:
     std::vector<CDFObjectClass *> cdfObjectList;
     void *cdfReaderPointer;
-    void *parentCDFObject;
+    CDFObject *parentCDFObject;
     bool _hasCustomReader;
 
   public:
@@ -97,8 +100,8 @@ namespace CDF {
     bool hasCustomReader() { return _hasCustomReader; }
 
     void setCDFReaderPointer(void *cdfReaderPointer) { this->cdfReaderPointer = cdfReaderPointer; }
-    void setParentCDFObject(void *parentCDFObject) { this->parentCDFObject = parentCDFObject; }
-    void *getParentCDFObject() const {
+    void setParentCDFObject(CDFObject *parentCDFObject) { this->parentCDFObject = parentCDFObject; }
+    CDFObject *getParentCDFObject() const {
       if (parentCDFObject == NULL) {
         throw(CDF_E_VARHASNOPARENT);
       }

Dockerfile

@@ -6,7 +6,7 @@ USER root
 LABEL maintainer="[email protected]"
 
 # Version should be same as in Definitions.h
-LABEL version="2.29.1"
+LABEL version="2.29.5"
 
 # Try to update image packages
 RUN apt-get -q -y update \

NEWS.md

@@ -1,3 +1,6 @@
+**Version 2.29.5 2025-02-14**
+- Timeseries are now sorted properly by key.
+
 **Version 2.29.2 2024-12-23**
 - Fixed issue with irregular grids for SSS SMOS L4 OI - LOPS-v2021 and Global Ocean - Coriolis Observation Re-Analysis CORA5.2. Added test (Issue #316)
 

adagucserverEC/CMakeJSONTimeSeries.cpp

@@ -11,7 +11,6 @@ const char *CMakeJSONTimeSeries::className = "CMakeJSONTimeSeries";
 int CMakeJSONTimeSeries::MakeJSONTimeSeries(CDrawImage *drawImage, CImageWarper *imageWarper, std::vector<CDataSource *> dataSources, int dataSourceIndex, int dX, int dY,
                                             CXMLParser::XMLElement *gfiStructure) {
   CDataSource *dataSource = dataSources[dataSourceIndex];
-
   CURUniqueRequests uniqueRequest;
   /**
    * DataPostProc: Here our datapostprocessor comes into action!
@@ -40,26 +39,11 @@ int CMakeJSONTimeSeries::MakeJSONTimeSeries(CDrawImage *drawImage, CImageWarper
     dataSource->setTimeStep(step);
     for (int dimnr = 0; dimnr < numberOfDims; dimnr++) {
       COGCDims *ogcDim = dataSource->requiredDims[dimnr];
-      uniqueRequest.set(dataSource->getFileName(), ogcDim->netCDFDimName.c_str(), dataSource->getDimensionIndex(dimnr), dataSource->getDimensionValue(dimnr));
+      uniqueRequest.addDimensionRangeRequest(dataSource->getFileName(), ogcDim->netCDFDimName.c_str(), dataSource->getDimensionIndex(dimnr), dataSource->getDimensionValue(dimnr).c_str());
     }
   }
 
-#ifdef CMakeJSONTimeSeries_DEBUG
-  CDBDebug("2): ***** sortAndAggregate *****");
-#endif
-  /* Sort it */
-  try {
-    uniqueRequest.sortAndAggregate();
-  } catch (int e) {
-    CDBError("Error in sortAndAggregate at line %d", e);
-    throw(__LINE__);
-  }
-
-#ifdef CMakeJSONTimeSeries_DEBUG
-  CDBDebug("3): ***** makeRequests and expanddata *****");
-#endif
-
-  /* Now efficiently fetch the data from the data files */
+  /* Bundle and make the requests */
   try {
     uniqueRequest.makeRequests(drawImage, imageWarper, dataSource, dX, dY, gfiStructure);
   } catch (int e) {

adagucserverEC/CUniqueRequests/CURTypes.h

@@ -3,33 +3,36 @@
 
 #include <map>
 #include "CTString.h"
+#include "CCDFObject.h"
 #define CCUniqueRequests_MAX_DIMS 255
 typedef std::map<int, std::string> map_type_dimvalindex;
 typedef std::map<int, std::string>::iterator it_type_dimvalindex;
 
 class CURUniqueRequests;
 
-struct CURAggregatedDimensionNoName {
-  int start;
-  std::vector<std::string> values;
-};
-struct CURAggregatedDimensionAndName {
-  CT::string name;
+struct StartAnValues {
   int start;
   std::vector<std::string> values;
 };
 
 struct CURDimInfo {
-  map_type_dimvalindex dimValuesMap;                          // All values, many starts with 1 count, result of set()
-  std::vector<CURAggregatedDimensionNoName> aggregatedValues; // Aggregated values (start/count series etc), result of  addDimSet()
+  map_type_dimvalindex dimValuesMap;           // All values, many starts with 1 count, result of set()
+  std::vector<StartAnValues> aggregatedValues; // Aggregated values (start/count series etc), result of  addDimSet()
 };
 
 typedef std::map<std::string, CURDimInfo> map_type_diminfo;
 typedef map_type_diminfo::iterator it_type_diminfo;
 
+struct CURAggregatedDimension {
+  CT::string name;
+  int start;
+  std::vector<std::string> values;
+};
+
 struct CURDimensionKey {
   std::string name;
-  int type;
+  CDF::Variable *cdfDimensionVariable;
+  bool isNumeric;
 };
 
 struct CURResult {
@@ -40,8 +43,10 @@ struct CURResult {
 };
 
 struct CURFileInfo {
-  std::vector<std::vector<CURAggregatedDimensionAndName>> requests;
+  std::vector<std::vector<CURAggregatedDimension>> requests;
   map_type_diminfo dimInfoMap; // AggregatedDimension name is key
 };
 
+typedef std::map<std::string, CURFileInfo>::iterator it_type_file;
+
 #endif

adagucserverEC/CUniqueRequests/CURUniqueRequests.cpp

@@ -8,9 +8,11 @@
 // #define CCUniqueRequests_DEBUG_HIGH
 const char *CURUniqueRequests::className = "CURUniqueRequests";
 
-int *CURUniqueRequests::getDimOrder() { return dimOrdering; }
+int *CURUniqueRequests::__getDimOrder() { return dimOrdering; }
 
-void CURUniqueRequests::set(const char *filename, const char *dimName, size_t dimIndex, CT::string dimValue) { fileInfoMap[filename].dimInfoMap[dimName].dimValuesMap[dimIndex] = dimValue.c_str(); }
+void CURUniqueRequests::addDimensionRangeRequest(const char *filename, const char *dimName, size_t dimIndex, std::string dimValue) {
+  fileInfoMap[filename].dimInfoMap[dimName].dimValuesMap[dimIndex] = dimValue;
+}
 
 void CURUniqueRequests::addDimSet(CURDimInfo &dimInfo, int start, std::vector<std::string> valueList) { dimInfo.aggregatedValues.push_back({.start = start, .values = valueList}); }
 
@@ -30,7 +32,7 @@ void CURUniqueRequests::nestRequest(it_type_diminfo diminfomapiterator, CURFileI
 #ifdef CCUniqueRequests_DEBUG_HIGH
     CDBDebug("Add request with following:");
 #endif
-    std::vector<CURAggregatedDimensionAndName> d;
+    std::vector<CURAggregatedDimension> d;
     for (int j = 0; j < depth; j++) {
       d.push_back(aggregatedDimensions[j]);
     }
@@ -113,7 +115,7 @@ void CURUniqueRequests::recurDataStructure(CXMLParser::XMLElement *dataStructure
       return recurDataStructure(dataStructure, result, depth + 1, dimOrdering, dimIndicesToSkip);
     }
   }
-  CT::string dimIndexName = result->dimensionKeys[dimIndex].name.c_str();
+  std::string dimIndexName = result->dimensionKeys[dimIndex].name;
 
   CXMLParser::XMLElement *el = NULL;
   try {
@@ -129,7 +131,7 @@ void CURUniqueRequests::recurDataStructure(CXMLParser::XMLElement *dataStructure
   }
 }
 
-void CURUniqueRequests::createStructure(CDataSource::DataObject *dataObject, CDrawImage *drawImage, CImageWarper *imageWarper, CDataSource *dataSource, int dX, int dY,
+void CURUniqueRequests::createStructure(std::vector<CURResult> results, CDataSource::DataObject *dataObject, CDrawImage *drawImage, CImageWarper *imageWarper, CDataSource *dataSource, int dX, int dY,
                                         CXMLParser::XMLElement *gfiStructure) {
 
   int numberOfDims = dataSource->requiredDims.size();
@@ -138,7 +140,7 @@ void CURUniqueRequests::createStructure(CDataSource::DataObject *dataObject, CDr
   layerStructure->add(CXMLParser::XMLElement("name", dataSource->getLayerName()));
 
   /* Add metadata */
-  CT::string standardName = dataObject->variableName.c_str();
+  std::string standardName = dataObject->variableName.c_str();
   CDF::Attribute *attr_standard_name = dataObject->cdfVariable->getAttributeNE("standard_name");
   if (attr_standard_name != NULL) {
     standardName = attr_standard_name->toString();
@@ -166,11 +168,8 @@ void CURUniqueRequests::createStructure(CDataSource::DataObject *dataObject, CDr
   }
 
   for (size_t i = 0; i < size_t(numberOfDims); i++) {
-
     if (std::find(dimIndicesToSkip.begin(), dimIndicesToSkip.end(), i) == dimIndicesToSkip.end()) {
-      CT::string dimNameToFind = dataSource->requiredDims[dimOrdering[i]]->name.c_str();
-
-      layerStructure->add(CXMLParser::XMLElement("dims", dimNameToFind.c_str()));
+      layerStructure->add(CXMLParser::XMLElement("dims", dataSource->requiredDims[dimOrdering[i]]->name.c_str()));
     }
   }
 
@@ -191,6 +190,7 @@ void CURUniqueRequests::createStructure(CDataSource::DataObject *dataObject, CDr
 }
 
 void CURUniqueRequests::makeRequests(CDrawImage *drawImage, CImageWarper *imageWarper, CDataSource *dataSource, int dX, int dY, CXMLParser::XMLElement *gfiStructure) {
+  sortAndAggregate();
 #ifdef CCUniqueRequests_DEBUG_HIGH
   CDBDebug("======================== makeRequests ========================================");
 #endif
@@ -228,7 +228,9 @@ void CURUniqueRequests::makeRequests(CDrawImage *drawImage, CImageWarper *imageW
   int numberOfDims = numberOfDataSourceDims + 2;
   size_t start[numberOfDims], count[numberOfDims];
   ptrdiff_t stride[numberOfDims];
-  CT::string dimName[numberOfDims];
+  std::string dimName[numberOfDims];
+
+  std::vector<CURResult> results;
 
   reader.open(dataSource, CNETCDFREADER_MODE_OPEN_HEADER);
 
@@ -237,9 +239,8 @@ void CURUniqueRequests::makeRequests(CDrawImage *drawImage, CImageWarper *imageW
 #endif
   for (size_t dataObjectNr = 0; dataObjectNr < dataSource->dataObjects.size(); dataObjectNr++) {
 
-    results.clear();
     CDataSource::DataObject *dataObject = dataSource->getDataObject(dataObjectNr);
-    CT::string variableName = dataObject->cdfVariable->name;
+    std::string variableName = dataObject->cdfVariable->name.c_str();
     // Show all requests
 
     for (it_type_file filemapiterator = fileInfoMap.begin(); filemapiterator != fileInfoMap.end(); filemapiterator++) {
@@ -413,15 +414,22 @@ void CURUniqueRequests::makeRequests(CDrawImage *drawImage, CImageWarper *imageW
 
                 // Fill in the dimension keys
                 for (size_t dataSourceDimIndex = 0; dataSourceDimIndex < dataSource->requiredDims.size(); dataSourceDimIndex++) {
-                  CT::string requestDimNameToFind = dataSource->requiredDims[dataSourceDimIndex]->netCDFDimName.c_str();
+                  std::string requestDimNameToFind = dataSource->requiredDims[dataSourceDimIndex]->netCDFDimName.c_str();
                   currentResultForIndex.dimensionKeys[dataSourceDimIndex].name = dataSource->requiredDims[dataSourceDimIndex]->value;
+                  auto dimVariable = variable->getParentCDFObject()->getVariableNE(requestDimNameToFind.c_str());
+                  auto isTime = dataSource->requiredDims[dataSourceDimIndex]->isATimeDimension;
+                  auto varType = dimVariable->getType();
+                  currentResultForIndex.dimensionKeys[dataSourceDimIndex].cdfDimensionVariable = dimVariable;
+                  currentResultForIndex.dimensionKeys[dataSourceDimIndex].isNumeric = isTime ? false : CDF::isCDFNumeric(varType);
+
                   int variableDimIndex = -1;
                   for (size_t d = 0; d < variable->dimensionlinks.size() - 2; d += 1) {
                     if (variable->dimensionlinks[d]->name.equals(requestDimNameToFind)) {
                       variableDimIndex = d;
                     }
                   }
                   if (variableDimIndex != -1) {
+
                     int requestDimIndex = -1;
                     for (size_t i = 0; i < request.size(); i++) {
                       if (request[i].name.equals(requestDimNameToFind.c_str())) {
@@ -435,8 +443,7 @@ void CURUniqueRequests::makeRequests(CDrawImage *drawImage, CImageWarper *imageW
                     auto values = request[requestDimIndex].values;
                     size_t numValues = values.size();
                     size_t multiplyIndex = multiplies[variableDimIndex];
-                    CT::string dimStr = values[(indexInVariable / multiplyIndex) % numValues].c_str();
-                    currentResultForIndex.dimensionKeys[dataSourceDimIndex].name = dimStr;
+                    currentResultForIndex.dimensionKeys[dataSourceDimIndex].name = values[(indexInVariable / multiplyIndex) % numValues];
                   }
                 }
 
@@ -470,7 +477,7 @@ void CURUniqueRequests::makeRequests(CDrawImage *drawImage, CImageWarper *imageW
       CDBDebug("**************** Create Structure ********************");
       CDBDebug("dataObjectNr: %d", dataObjectNr);
 #endif
-      createStructure(dataObject, drawImage, imageWarper, dataSource, dX, dY, gfiStructure);
+      createStructure(results, dataObject, drawImage, imageWarper, dataSource, dX, dY, gfiStructure);
     } catch (int e) {
       CDBError("Error in createStructure at line %d", e);
       throw(__LINE__);

adagucserverEC/CUniqueRequests/CURUniqueRequests.h

@@ -15,26 +15,25 @@
 
 class CURUniqueRequests {
 
-  typedef std::map<std::string, CURFileInfo>::iterator it_type_file;
-
 private:
   DEF_ERRORFUNCTION();
   std::map<std::string, CURFileInfo> fileInfoMap;
   int dimOrdering[CCUniqueRequests_MAX_DIMS];
-  std::vector<CURResult> results;
-  CURAggregatedDimensionAndName aggregatedDimensions[CCUniqueRequests_MAX_DIMS];
+  CURAggregatedDimension aggregatedDimensions[CCUniqueRequests_MAX_DIMS];
 
   void recurDataStructure(CXMLParser::XMLElement *dataStructure, CURResult *result, int depth, int *dimOrdering, std::vector<int> dimIndicesToSkip);
-  void createStructure(CDataSource::DataObject *dataObject, CDrawImage *drawImage, CImageWarper *imageWarper, CDataSource *dataSource, int dX, int dY, CXMLParser::XMLElement *gfiStructure);
+  void createStructure(std::vector<CURResult> results, CDataSource::DataObject *dataObject, CDrawImage *drawImage, CImageWarper *imageWarper, CDataSource *dataSource, int dX, int dY,
+                       CXMLParser::XMLElement *gfiStructure);
   void addDimSet(CURDimInfo &dimInfo, int start, std::vector<std::string> valueList);
   void nestRequest(it_type_diminfo diminfomapiterator, CURFileInfo &fileInfo, int depth);
+  void sortAndAggregate();
 
 public:
   bool readDataAsCDFDouble = false;
-  int *getDimOrder();
+  int *__getDimOrder();
+
+  void addDimensionRangeRequest(const char *filename, const char *dimName, size_t dimIndex, std::string dimValue);
   void makeRequests(CDrawImage *drawImage, CImageWarper *imageWarper, CDataSource *dataSource, int dX, int dY, CXMLParser::XMLElement *gfiStructure);
-  void set(const char *filename, const char *dimName, size_t dimIndex, CT::string dimValue);
-  void sortAndAggregate();
 };
 
 #endif

adagucserverEC/CUniqueRequests/requestUtils.cpp

@@ -12,11 +12,13 @@ bool sortDimensionKeysRecursive(CURResult &result1, CURResult &result2, int dept
     d = sortDimensionKeysRecursive(result1, result2, depth + 1);
   }
 
-  int *dimOrder = result1.parent->getDimOrder();
+  int *dimOrder = result1.parent->__getDimOrder();
   int dimOrderIndex = dimOrder[depth];
-  std::string s1 = result1.dimensionKeys[dimOrderIndex].name;
-  std::string s2 = result2.dimensionKeys[dimOrderIndex].name;
-  padTo(s1, 8);
-  padTo(s2, 8);
-  return s1.compare(s2) < 0 || d;
+  if (result1.dimensionKeys[dimOrderIndex].isNumeric) {
+    double n1 = std::stod(result1.dimensionKeys[dimOrderIndex].name);
+    double n2 = std::stod(result2.dimensionKeys[dimOrderIndex].name);
+    return n1 < n2 || d;
+  } else {
+    return result1.dimensionKeys[dimOrderIndex].name.compare(result2.dimensionKeys[dimOrderIndex].name) < 0 || d;
+  }
 }

adagucserverEC/Definitions.h

@@ -28,7 +28,7 @@
 #ifndef Definitions_H
 #define Definitions_H
 
-#define ADAGUCSERVER_VERSION "2.29.1" // Please also update in the Dockerfile to the same version
+#define ADAGUCSERVER_VERSION "2.29.5" // Please also update in the Dockerfile to the same version
 
 // CConfigReaderLayerType
 #define CConfigReaderLayerTypeUnknown 0