dataset.h

Go to the documentation of this file.

/* Copyright (C) 2017-2024 Chris Piker <chris-piker@uiowa.edu>
 *
 * This file is part of das2C, the Core Das2 C Library.
 *
 * das2C is free software; you can redistribute it and/or modify it under
 * the terms of the GNU Lesser General Public License version 2.1 as published
 * by the Free Software Foundation.
 *
 * das2C is distributed in the hope that it will be useful, but WITHOUT ANY
 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 * FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for
 * more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * version 2.1 along with das2C; if not, see <http://www.gnu.org/licenses/>.
 */
 
 
#ifndef _das_dataset_h_
#define _das_dataset_h_
 
#include <das3/dimension.h>
#include <das3/codec.h>
 
#ifdef __cplusplus
extern "C" {
#endif
 
/* Old initial comment that kicked off the entire das2 data model design...
 *
 * The structures below are the start of an idea on how to get independent
 * parameters for data at any particular index.  These are just thoughts
 * at the moment and don't affect any working code.  There are many ways
 * to do this.  The CDF and QStream assumption is that there are the same
 * number of parameters locating a data point in parameter space as there
 * are indices to the dataset.  Because of this x,y,z scatter data are 
 * hard to handle.
 *  
 * For x,y,z scatter lists there is 1 index for any point in the dataset, 
 * but for each index there are 2 independent parameters.  Basically QStream
 * and CDF assume that all datasets are CUBEs in parameter space but this
 * is not the case for a great many sets. 
 *
 * To adequately handle these 'path' datasets a parameter map is required.
 * The mapping takes 1 index value per data rank and returns 1 to N parameter
 * values.
 *
 * These structures start to handle this idea but are just doodles at this
 * point. -cwp 2017-07-25
 */
 
/* Second comment that added desire for flexible data types ...
 * 
 * Thinking about coordinate returns, how about a data set of thefts / month
 * in 5 cities...  Won't usually come up, but should be possible
 * to handle.  Here's the data set:
 *
 *              2016-01 2016-02 2016-03 2016-04 2016-05 2016-06
 * Baltimore       2351    3789    4625    5525    6135    5902
 * Bogotaà      109065  110365   99625   98265   43850   33892
 * Chicago         4789    5764    8901   10145   13456   22678
 * Des Moines         4      10      33      35      44     107
 *
 * Properties:  Title -> "Thefts/Month for selected cities"
 *
 * Okay, the X axis data type is text[12] (need null char)
 *           Y axis data type is datetime
 *           Z axis data type is datum, "thefts month**-1"
 *
 * So what is the return value from pDs->bin(pDs, 0, 0) ?
 *
 * The bin is defined on the space of all UTC times, and on the space of all
 * cities in the data set.
 *
 *
 * So, what about this common data set, interference events:
 *
 * |<----------     Bin      ------>|  |<----- Value --->|
 * 2016-01-01T14:00  2016-01-02T02:20  Mag Roll
 * 2016-01-01T15:40  2016-01-01T15:41  Stabilization Pulse
 * 2016-01-01T15:43  2016-01-01T15:44  Stabilization Pulse
 * 2016-01-01T15:45  2016-01-01T15:47  Stabilization Pulse
 * 2016-01-01T15:48  2016-01-01T15:50  Stabilization Pulse
 *
 * So what is the return value from pDs->bin(pDs, 0) ?
 *
 * The space is UTC time,  So each bin start and stop is defined on the space
 * of all UTC times.
 * -cwp 2017-??-??
 */
 
/* Number of encoders that can be stored internally, more then this and they
 * have to be allocated on the heap. 
 *
 * Current test case with the most codec/dateste is ex21, at 7.
 * If you bump this make a test case with one more variable than the new
 * value below.
 */
#define DASDS_LOC_ENC_SZ 6
 
/* Small vector size for array and dimension pointer sets. Larger then
 * DASDS_LOC_ENC_SZ  because these are just sizeof(void*) each not ~200 B.
 *
 * NOTE: No test case reaches these sizes, but both were reduced to 6 for 
 *       a onetime test against ex21_tracers_cdpu_status.d3b (uDims=7, 
 *       uArrays=7), on 2026-07-16.  
 *
 * If you touch the dynamic #of array or dims code, drop these back to 6
 * each and run `make test`.
 */
#define DASDS_LOC_ARY_SZ 32
#define DASDS_LOC_DIM_SZ 32
 
typedef struct dataset {
   DasDesc base;        /* This would be equivalent to the properties for 
                           a packet descriptor.  Typically in das 2.2 packets
                           don't have a descriptor, only streams and planes
                           but access to the stream descriptor forwards through
                           here. */
   
   int nRank;           /* The number of whole-dataset index dimensions. 
                         * Variables can define internal dimensions but they
                         * can't use indices in the first nRank positions for
                         * internal use, as these are used to correlate values
                         * across the dataset. */
   
   /* A text identifier for this instance of a data set */
   char sId[DAS_MAX_ID_BUFSZ];
   
   /* A text identifier for the join group for this dataset.  Datasets with
    * the same groupID should be joined automatically by display clients. 
    */
   char sGroupId[DAS_MAX_ID_BUFSZ];
                        
   size_t uDims;        /* Number of dimensions, das datasets are 
                          * implicitly bundles in qdataset terms. */
   
   DasDim** lDims;      /* The data variable object arrays, internal or external */
   size_t uSzDims;      /* lDims memory size, internal or external */
 
   DasDim* aDims[DASDS_LOC_DIM_SZ];  /* small vector memory */
   
   size_t uArrays;      /* The number of low-level arrays */
   DasAry** lArrays;    /* Array objects, internal or external */
   size_t uSzArrays;    /* lArrays memory size, internal or external */
 
   DasAry* aArrays[DASDS_LOC_ARY_SZ];  /* small vector memory */
   
   ptrdiff_t _shape[VARIDX_MAX];  /* cache shape calls for speed */
   
   bool _dynamic;      /* If true, the dataset may still be changing and all
                          bulk properties such as the iteration shape should be
                          recalculated instead of using cached values. 
                          If false, cached values are expected to already be 
                          available */
 
   /* dataset arrays can be written in chunks to output buffers. The number of
    * elements in each chuck, the encoding of each element any separators are
    * defined below. */
   /* DasCodec** lEncs; */
 
   size_t uCodecs;   /* Number of valid codecs */
 
   /* These become large vector memory when uSzEncs > DASDS_LOC_ENC_SZ */
 
   /* When the number of valid codecs grows past DASDS_LOC_ENC_SZ, use an
      external buffer for all of them */
 
   DasCodec*  lCodecs;   /* Codec pointers, internal or external */
   int*       lItems;    /* Number of items to decode per codec, internal or ex */
   size_t     uSzCodecs; /* Codec & Item array memory size, internal or external */
   
   DasCodec aCodecs[DASDS_LOC_ENC_SZ];  /* small vector memory */
   int aItems[DASDS_LOC_ENC_SZ];      /* small vector memory */
 
   /* Set to true when encode is called, make sure data doesn't go 
    * out the door unless the descriptor is sent first */
   bool bSentHdr;
 
   void* pUser;
 
} DasDs;
 
DAS_API DasDs* new_DasDs(
   const char* sId, const char* sGroupId, int nRank
);
 
DAS_API DasDs* DasDs_copy(const DasDs* pThis);
 
DAS_API DasErrCode DasDs_replaceAry(DasDs* pThis, const char* sOldId, DasAry* pNew);
 
DAS_API void del_DasDs(DasDs* pThis);
 
DAS_API void DasDs_setMutable(DasDs* pThis, bool bChangeAllowed);
 
 
#define DasDs_mutable(P) P->_mutable
 
DAS_API int DasDs_shape(const DasDs* pThis, ptrdiff_t* pShape);
 
DAS_API ptrdiff_t DasDs_lengthIn(const DasDs* pThis, int nIdx, ptrdiff_t* pLoc);
 
 
#define DasDs_group(P) ((const char*)(P)->sGroupId)
 
#define DasDs_id(P) ((const char*)(P)->sId)
 
 
#define DasDs_rank(P) ((P)->nRank)
 
DAS_API DasErrCode DasDs_addAry(DasDs* pThis, DasAry* pAry);
 
 
#define DasDs_numAry(P) ((P)->uArrays)
 
#define DasDs_getAry(P, I) ((P)->lArrays[(I)])
 
 
DAS_API DasAry* DasDs_getAryById(DasDs* pThis, const char* sAryId);
 
DAS_API size_t DasDs_memUsed(const DasDs* pThis);
 
DAS_API size_t DasDs_memIndexed(const DasDs* pThis);
 
DAS_API size_t DasDs_memOwned(const DasDs* pThis);
 
#define DasDs_numCodecs( pThis )  ( (pThis)->uCodecs )
 
#define DasDs_getCodec( pThis, I )  ( &( (pThis)->lCodecs[(I)] ) )
 
DAS_API DasCodec* DasDs_getCodecFor(
   const DasDs* pThis, const char* sAryId, int* pItems
);
 
#define DasDs_pktItems( P, I ) ( (P)->lItems[(I)] )
 
DAS_API DasCodec* DasDs_addFixedCodec(
   DasDs* pThis, const char* sAryId, const char* sSemantic, 
   const char* sEncType, int nItemBytes, int nNumItems, bool bRead
);
 
DAS_API DasCodec* DasDs_addStringCodec(
    DasDs* pThis, const char* sAryId, const char* sSemantic, 
    const char* sEncType, int nItemTerm, ubyte uTerm, int nNumItems,
    bool bRead
);
 
 
DAS_API DasCodec* DasDs_addCodecFrom(
   DasDs* pThis, const char* sAryId, const DasCodec* pOther, int nNumItems,
   bool bRead
);
 
DAS_API DasDs* new_DasDs_xml(DasBuf* pBuf, DasDesc* pParent, int nPktId);
 
DAS_API DasErrCode DasDs_encodeHdr(DasDs* pThis, DasBuf* pBuf);
 
DAS_API DasErrCode DasDs_decodeData(DasDs* pThis, DasBuf* pBuf);
 
DAS_API DasErrCode DasDs_encodeData(DasDs* pThis, DasBuf* pBuf, ptrdiff_t iIdx0);
 
 
DAS_API int DasDs_recBytes(const DasDs* pThis);
 
 
DAS_API size_t DasDs_clearRagged0(DasDs* pThis);
 
 
DAS_API DasDim* DasDs_makeDim(
   DasDs* pThis, enum dim_type dType, const char* sDim, const char* sId
);
 
DAS_API DasErrCode DasDs_addDim(DasDs* pThis, DasDim* pDim);
 
DAS_API size_t DasDs_numDims(const DasDs* pThis, enum dim_type dmt);
 
 
DAS_API DasDim* DasDs_getDim(
   DasDs* pThis, const char* sDim, enum dim_type dmt
);
 
DAS_API DasDim* DasDs_getDimByIdx(
   DasDs* pThis, size_t idx, enum dim_type vt
);
 
DAS_API DasDim* DasDs_getDimById(DasDs* pThis, const char* sId);
 
 
DAS_API char* DasDs_toStr(const DasDs* pThis, char* sBuf, int nLen);
 
 
bool DasDs_cubicCoords(const DasDs* pThis, const DasDim** pCoords);
 
 
 
 
/* Ideas I'm still working on...
 
bool DataGen_grid(const DataSet* pDataset);
 
const DataSet** Dg_griddedIn(const DataSet* pDataset);
 
 
/ * The two functions below are really useful but I'll need to crack open
   a double pack of Flex and Bison to get it done so I'm punting for now. * /
    
/ * Ex Expression:  $spec_dens[i][j][k] * /
const Function* Dataset_evalDataExp(Dataset* pThis, const char* sExpression);
 
/ * Ex Expression:  $craft_alt[i][j] - 0.5 * $delay_time[k] * 299792 * /
const Function* Dataset_evalCoordExp(Dataset* pThis, const char* sExpression);
 
 
/ ** Get the coefficients for iterating over a 1-D slice of a regular (i.e. 
 * non-ragged) dataset.
 *
 * This function dose not work for ragged datasets and merely returns NULL if
 * asked for iteration coefficients for such a set.  In such a case use 
 * Dataset_copySlice1D().
 *
 * /
const void* Dataset_slice1D(
   const Dataset* pThis, const char* sDs, const char* sCoord, int iCoordIdx,
   int* pCoeff
);
 
/ ** Increment the reference count on any array objects that are part of 
 * a data space.
 *
 * This is useful in instances where the underlying data arrays are going
 * to be represented by an organizational structure other than datasets
 * and DataSets since Das array objects only free data memory if their
 * reference count is zero.
 *
 * @param pThis
 * /
void DataSpace_incAryRef(Dataset* pThis);
 
/ * Need a way to trigger callbacks from datasets changing, not just
   packets changing.  It could be useful to work on items from the
   dataset level instead of just the packet level * /
 bool DataSpace_stream(Dataset* pThis); * /
 
 
 
/ ** Indicate the physical degrees of freedom for a dataset by denoting a
 * complete list of coordinate sets.  
 *
 * A list of coordinates over which an entire dataset is defined is called
 * a span.  Datasets may have 1-N spans.
 * /
int DataSpace_addSpan(const char* sDsId, const char** lCoords, size_t nCoords);
*/
 
#ifdef __cplusplus
}
#endif
 
#endif /* _das_dataset_h */