NetCDF  4.8.1
zarr.c
1 /*********************************************************************
2  * Copyright 2018, UCAR/Unidata
3  * See netcdf/COPYRIGHT file for copying and redistribution conditions.
4  *********************************************************************/
5 
6 #include "zincludes.h"
7 
8 /**************************************************/
9 /* Forwards */
10 
11 static int applycontrols(NCZ_FILE_INFO_T* zinfo);
12 
13 /***************************************************/
14 /* API */
15 
24 int
25 ncz_create_dataset(NC_FILE_INFO_T* file, NC_GRP_INFO_T* root, const char** controls)
26 {
27  int stat = NC_NOERR;
28  NCZ_FILE_INFO_T* zinfo = NULL;
29  NCZ_GRP_INFO_T* zgrp = NULL;
30  NCURI* uri = NULL;
31  NC* nc = NULL;
32  NCjson* json = NULL;
33  char* key = NULL;
34 
35  ZTRACE(3,"file=%s root=%s controls=%s",file->hdr.name,root->hdr.name,(controls?nczprint_envv(controls):"null"));
36 
37  nc = (NC*)file->controller;
38 
39  /* Add struct to hold NCZ-specific file metadata. */
40  if (!(zinfo = calloc(1, sizeof(NCZ_FILE_INFO_T))))
41  {stat = NC_ENOMEM; goto done;}
42  file->format_file_info = zinfo;
43  zinfo->common.file = file;
44 
45  /* Add struct to hold NCZ-specific group info. */
46  if (!(zgrp = calloc(1, sizeof(NCZ_GRP_INFO_T))))
47  {stat = NC_ENOMEM; goto done;}
48  root->format_grp_info = zgrp;
49  zgrp->common.file = file;
50 
51  /* Fill in NCZ_FILE_INFO_T */
52  zinfo->created = 1;
53  zinfo->common.file = file;
54  zinfo->native_endianness = (NCZ_isLittleEndian() ? NC_ENDIAN_LITTLE : NC_ENDIAN_BIG);
55  if((zinfo->envv_controls=NCZ_clonestringvec(0,controls)) == NULL)
56  {stat = NC_ENOMEM; goto done;}
57 
58  /* fill in some of the zinfo and zroot fields */
59  zinfo->zarr.zarr_version = atoi(ZARRVERSION);
60  sscanf(NCZARRVERSION,"%lu.%lu.%lu",
61  &zinfo->zarr.nczarr_version.major,
62  &zinfo->zarr.nczarr_version.minor,
63  &zinfo->zarr.nczarr_version.release);
64 
65  /* Apply client controls */
66  if((stat = applycontrols(zinfo))) goto done;
67 
68  /* Load auth info from rc file */
69  if((stat = ncuriparse(nc->path,&uri))) goto done;
70  if(uri) {
71  if((stat = NC_authsetup(&zinfo->auth, uri)))
72  goto done;
73  }
74 
75  /* initialize map handle*/
76  if((stat = nczmap_create(zinfo->controls.mapimpl,nc->path,nc->mode,zinfo->controls.flags,NULL,&zinfo->map)))
77  goto done;
78 
79 done:
80  ncurifree(uri);
81  NCJreclaim(json);
82  nullfree(key);
83  return ZUNTRACE(stat);
84 }
85 
94 int
95 ncz_open_dataset(NC_FILE_INFO_T* file, const char** controls)
96 {
97  int stat = NC_NOERR;
98  NC* nc = NULL;
99  NC_GRP_INFO_T* root = NULL;
100  NCURI* uri = NULL;
101  void* content = NULL;
102  NCjson* json = NULL;
103  NCZ_FILE_INFO_T* zinfo = NULL;
104  int mode;
105  NClist* modeargs = NULL;
106  char* nczarr_version = NULL;
107  char* zarr_format = NULL;
108 
109  ZTRACE(3,"file=%s controls=%s",file->hdr.name,(controls?nczprint_envv(controls):"null"));
110 
111  /* Extract info reachable via file */
112  nc = (NC*)file->controller;
113  mode = nc->mode;
114 
115  root = file->root_grp;
116  assert(root != NULL && root->hdr.sort == NCGRP);
117 
118  /* Add struct to hold NCZ-specific file metadata. */
119  if (!(file->format_file_info = calloc(1, sizeof(NCZ_FILE_INFO_T))))
120  {stat = NC_ENOMEM; goto done;}
121  zinfo = file->format_file_info;
122 
123  /* Fill in NCZ_FILE_INFO_T */
124  zinfo->created = 0;
125  zinfo->common.file = file;
126  zinfo->native_endianness = (NCZ_isLittleEndian() ? NC_ENDIAN_LITTLE : NC_ENDIAN_BIG);
127  if((zinfo->envv_controls = NCZ_clonestringvec(0,controls))==NULL) /*0=>envv style*/
128  {stat = NC_ENOMEM; goto done;}
129 
130  /* Add struct to hold NCZ-specific group info. */
131  if (!(root->format_grp_info = calloc(1, sizeof(NCZ_GRP_INFO_T))))
132  {stat = NC_ENOMEM; goto done;}
133  ((NCZ_GRP_INFO_T*)root->format_grp_info)->common.file = file;
134 
135  /* Apply client controls */
136  if((stat = applycontrols(zinfo))) goto done;
137 
138  /* initialize map handle*/
139  if((stat = nczmap_open(zinfo->controls.mapimpl,nc->path,mode,zinfo->controls.flags,NULL,&zinfo->map)))
140  goto done;
141 
142  if((stat = ncz_read_superblock(file,&nczarr_version,&zarr_format))) goto done;
143 
144  if(nczarr_version == NULL) /* default */
145  nczarr_version = strdup(NCZARRVERSION);
146  if(zarr_format == NULL) /* default */
147  zarr_format = strdup(ZARRVERSION);
148  /* Extract the information from it */
149  if(sscanf(zarr_format,"%d",&zinfo->zarr.zarr_version)!=1)
150  {stat = NC_ENCZARR; goto done;}
151  if(sscanf(nczarr_version,"%lu.%lu.%lu",
152  &zinfo->zarr.nczarr_version.major,
153  &zinfo->zarr.nczarr_version.minor,
154  &zinfo->zarr.nczarr_version.release) == 0)
155  {stat = NC_ENCZARR; goto done;}
156 
157  /* Load auth info from rc file */
158  if((stat = ncuriparse(nc->path,&uri))) goto done;
159  if(uri) {
160  if((stat = NC_authsetup(&zinfo->auth, uri)))
161  goto done;
162  }
163 
164 done:
165  nullfree(zarr_format);
166  nullfree(nczarr_version);
167  ncurifree(uri);
168  nclistfreeall(modeargs);
169  if(json) NCJreclaim(json);
170  nullfree(content);
171  return ZUNTRACE(stat);
172 }
173 
184 int
185 NCZ_isnetcdf4(struct NC_FILE_INFO* h5)
186 {
187  int isnc4 = 1;
188  NC_UNUSED(h5);
189  return isnc4;
190 }
191 
204 int
205 NCZ_get_libversion(unsigned long* majorp, unsigned long* minorp,unsigned long* releasep)
206 {
207  unsigned long m0,m1,m2;
208  sscanf(NCZARRVERSION,"%lu.%lu.%lu",&m0,&m1,&m2);
209  if(majorp) *majorp = m0;
210  if(minorp) *minorp = m1;
211  if(releasep) *releasep = m2;
212  return NC_NOERR;
213 }
214 
226 int
227 NCZ_get_superblock(NC_FILE_INFO_T* file, int* superblockp)
228 {
229  NCZ_FILE_INFO_T* zinfo = file->format_file_info;
230  if(superblockp) *superblockp = zinfo->zarr.nczarr_version.major;
231  return NC_NOERR;
232 }
233 
234 /**************************************************/
235 /* Utilities */
236 
237 #if 0
238 
245 static int
246 ncz_open_rootgroup(NC_FILE_INFO_T* dataset)
247 {
248  int stat = NC_NOERR;
249  int i;
250  NCZ_FILE_INFO_T* zfile = NULL;
251  NC_GRP_INFO_T* root = NULL;
252  void* content = NULL;
253  char* rootpath = NULL;
254  NCjson* json = NULL;
255 
256  ZTRACE(3,"dataset=",dataset->hdr.name);
257 
258  zfile = dataset->format_file_info;
259 
260  /* Root should already be defined */
261  root = dataset->root_grp;
262 
263  assert(root != NULL);
264 
265  if((stat=nczm_concat(NULL,ZGROUP,&rootpath)))
266  goto done;
267  if((stat = NCZ_downloadjson(zfile->map, rootpath, &json)))
268  goto done;
269  /* Process the json */
270  for(i=0;i<nclistlength(json->contents);i+=2) {
271  const NCjson* key = nclistget(json->contents,i);
272  const NCjson* value = nclistget(json->contents,i+1);
273  if(strcmp(NCJstring(key),"zarr_format")==0) {
274  int zversion;
275  if(sscanf(NCJstring(value),"%d",&zversion)!=1)
276  {stat = NC_ENOTNC; goto done;}
277  /* Verify against the dataset */
278  if(zversion != zfile->zarr.zarr_version)
279  {stat = NC_ENOTNC; goto done;}
280  }
281  }
282 
283 done:
284  if(json) NCJreclaim(json);
285  nullfree(rootpath);
286  nullfree(content);
287  return ZUNTRACE(stat);
288 }
289 #endif
290 
291 #if 0
292 
301 int
302 ncz_unload_jatts(NCZ_FILE_INFO_T* zinfo, NC_OBJ* container, NCjson* jattrs, NCjson* jtypes)
303 {
304  int stat = NC_NOERR;
305  char* fullpath = NULL;
306  char* akey = NULL;
307  char* tkey = NULL;
308  NCZMAP* map = zinfo->map;
309 
310  assert((jattrs->sort = NCJ_DICT));
311  assert((jtypes->sort = NCJ_DICT));
312 
313  if(container->sort == NCGRP) {
314  NC_GRP_INFO_T* grp = (NC_GRP_INFO_T*)container;
315  /* Get grp's fullpath name */
316  if((stat = NCZ_grpkey(grp,&fullpath)))
317  goto done;
318  } else {
319  NC_VAR_INFO_T* var = (NC_VAR_INFO_T*)container;
320  /* Get var's fullpath name */
321  if((stat = NCZ_varkey(var,&fullpath)))
322  goto done;
323  }
324 
325  /* Construct the path to the .zattrs object */
326  if((stat = nczm_concat(fullpath,ZATTRS,&akey)))
327  goto done;
328 
329  /* Always write as V2 */
330 
331  {
332  NCjson* k = NULL;
333  NCjson* v = NULL;
334  /* remove any previous version */
335  if(NCJremove(jattrs,NCZ_V2_ATTRS,&k,&v) == NC_NOERR) {
336  NCJreclaim(k); NCJreclaim(v);
337  }
338  }
339 
340  if(!(zinfo->controls.flags & FLAG_PUREZARR)) {
341  /* Insert the jtypes into the set of attributes */
342  if((stat = NCJinsert(jattrs,NCZ_V2_ATTRS,jtypes))) goto done;
343  }
344 
345  /* Upload the .zattrs object */
346  if((stat=NCZ_uploadjson(map,tkey,jattrs)))
347  goto done;
348 
349 done:
350  if(stat) {
351  NCJreclaim(jattrs);
352  NCJreclaim(jtypes);
353  }
354  nullfree(fullpath);
355  nullfree(akey);
356  nullfree(tkey);
357  return stat;
358 }
359 #endif
360 
361 static const char*
362 controllookup(const char** envv_controls, const char* key)
363 {
364  const char** p;
365  for(p=envv_controls;*p;p+=2) {
366  if(strcasecmp(key,*p)==0) {
367  return p[1];
368  }
369  }
370  return NULL;
371 }
372 
373 
374 static int
375 applycontrols(NCZ_FILE_INFO_T* zinfo)
376 {
377  int i,stat = NC_NOERR;
378  const char* value = NULL;
379  NClist* modelist = nclistnew();
380  int noflags = 0; /* track non-default negative flags */
381 
382  if((value = controllookup((const char**)zinfo->envv_controls,"mode")) != NULL) {
383  if((stat = NCZ_comma_parse(value,modelist))) goto done;
384  }
385  /* Process the modelist first */
386  zinfo->controls.mapimpl = NCZM_DEFAULT;
387  for(i=0;i<nclistlength(modelist);i++) {
388  const char* p = nclistget(modelist,i);
389  if(strcasecmp(p,PUREZARRCONTROL)==0) zinfo->controls.flags |= (FLAG_PUREZARR|FLAG_XARRAYDIMS);
390  else if(strcasecmp(p,XARRAYCONTROL)==0) zinfo->controls.flags |= (FLAG_XARRAYDIMS|FLAG_PUREZARR); /*xarray=>zarr*/
391  else if(strcasecmp(p,NOXARRAYCONTROL)==0) {
392  noflags |= FLAG_XARRAYDIMS;
393  zinfo->controls.flags |= FLAG_PUREZARR; /*noxarray=>zarr*/
394  }
395  else if(strcasecmp(p,"zip")==0) zinfo->controls.mapimpl = NCZM_ZIP;
396  else if(strcasecmp(p,"file")==0) zinfo->controls.mapimpl = NCZM_FILE;
397  else if(strcasecmp(p,"s3")==0) zinfo->controls.mapimpl = NCZM_S3;
398  }
399  /* Apply negative controls by turning off negative flags */
400  /* This is necessary to avoid order dependence of mode flags when both positive and negative flags are defined */
401  zinfo->controls.flags &= (~noflags);
402 
403  /* Process other controls */
404  if((value = controllookup((const char**)zinfo->envv_controls,"log")) != NULL) {
405  zinfo->controls.flags |= FLAG_LOGGING;
406  ncsetlogging(1);
407  }
408  if((value = controllookup((const char**)zinfo->envv_controls,"show")) != NULL) {
409  if(strcasecmp(value,"fetch")==0)
410  zinfo->controls.flags |= FLAG_SHOWFETCH;
411  }
412 done:
413  nclistfreeall(modelist);
414  return stat;
415 }
416 
NC_NOERR
#define NC_NOERR
No Error.
Definition: netcdf.h:333
NC_ENDIAN_BIG
#define NC_ENDIAN_BIG
In HDF5 files you can set the endianness of variables with nc_def_var_endian().
Definition: netcdf.h:291
NC_ENCZARR
#define NC_ENCZARR
Error at NCZarr layer.
Definition: netcdf.h:483
NC_ENDIAN_LITTLE
#define NC_ENDIAN_LITTLE
In HDF5 files you can set the endianness of variables with nc_def_var_endian().
Definition: netcdf.h:290
NC_ENOTNC
#define NC_ENOTNC
Not a netcdf file.
Definition: netcdf.h:389
NC_ENOMEM
#define NC_ENOMEM
Memory allocation (malloc) failure.
Definition: netcdf.h:413