NetCDF  4.8.1
filter_example.c
Go to the documentation of this file.
1 /*
2  Copyright 2018, UCAR/Unidata
3  See COPYRIGHT file for copying and redistribution conditions.
4 */
5 /*
6 This file is the same as nc_test4/test_filter.c
7 */
8 
44 #include "config.h"
45 #include <stdio.h>
46 #include <string.h>
47 #include <stdlib.h>
48 
49 #include <hdf5.h>
50 #include "netcdf.h"
51 #include "netcdf_filter.h"
52 
53 /* The HDF assigned id for bzip compression */
54 #define BZIP2_ID 307
55 /* The compression level used in this example */
56 #define BZIP2_LEVEL 9
57 
58 #define TESTFILE "bzip2.nc"
59 
60 /* Point at which we give up */
61 #define MAXERRS 8
62 
63 #define NDIMS 4
64 #define DIMSIZE 4
65 #define CHUNKSIZE 4 /* Note: not the total size of the chunk, but size wrt a dim*/
66 
67 static size_t dimsize = DIMSIZE;
68 static size_t chunksize = CHUNKSIZE;
69 static size_t actualdims = NDIMS;
70 
71 static size_t actualproduct = 1; /* x-product over dim sizes */
72 static size_t chunkproduct = 1; /* x-product over chunksizes */
73 
74 static size_t dims[NDIMS];
75 static size_t chunks[NDIMS];
76 
77 static int nerrs = 0;
78 
79 static int ncid, varid;
80 static int dimids[NDIMS];
81 static float* array = NULL;
82 static float* expected = NULL;
83 static unsigned int filterid = 0;
84 static unsigned int* params = NULL;
85 
86 /* Forward */
87 static void init(int argc, char** argv);
88 static int test_bzip2(void);
89 static int verifychunks(void);
90 
91 #define ERRR do { \
92 fflush(stdout); /* Make sure our stdout is synced with stderr. */ \
93 fprintf(stderr, "Sorry! Unexpected result, %s, line: %d\n", \
94  __FILE__, __LINE__); \
95 nerrs++;\
96 } while (0)
97 
98 static int
99 check(int err,int line)
100 {
101  if(err != NC_NOERR) {
102  fprintf(stderr,"fail (%d): %s\n",line,nc_strerror(err));
103  fflush(stderr);
104  exit(1);
105  }
106  return NC_NOERR;
107 }
108 
109 #define CHECK(x) check(x,__LINE__)
110 
111 /*
112 Read the chunking information about the variable
113 and verify that it is as expected.
114 */
115 
116 static int
117 verifychunks(void)
118 {
119  int i;
120  int store = -1;
121  size_t chunksizes[NDIMS];
122  memset(chunksizes,0,sizeof(chunksizes));
123  CHECK(nc_inq_var_chunking(ncid, varid, &store, chunksizes));
124  /* Storate must be chunked, not contiguous */
125  if(store != NC_CHUNKED) {
126  fprintf(stderr,"bad chunk store\n");
127  return NC_ESTORAGE;
128  }
129  /* Chunk sizes must match our predefined set */
130  for(i=0;i<actualdims;i++) {
131  if(chunksizes[i] != chunks[i]) {
132  fprintf(stderr,"bad chunk size: %d\n",i);
133  return NC_EBADCHUNK;
134  }
135  }
136  return 1;
137 }
138 
139 /*
140 Compare the data we wrote against the data we read.
141 */
142 
143 static int
144 compare(void)
145 {
146  int errs = 0;
147  int i;
148  printf("data comparison: |array|=%ld\n",(unsigned long)actualproduct);
149  for(i=0;i<actualproduct;i++) {
150  if(expected[i] != array[i]) {
151  printf("mismatch: array[%d]=%f expected[%d]=%f\n",
152  i,array[i],i,expected[i]);
153  errs++;
154  if(errs >= MAXERRS)
155  break;
156  }
157  }
158  if(errs == 0)
159  printf("no data errors\n");
160  if(actualproduct <= 1)
161  return NC_EBADDIM;
162  return (errs == 0 ? NC_NOERR: NC_EINVAL);
163 }
164 
165 /*
166 Create the file, write it, then re-read for comparison.
167 */
168 static int
169 test_bzip2(void)
170 {
171  int i;
172  unsigned int level = BZIP2_LEVEL;
173  size_t nparams = 0;
174 
175  printf("\n*** Testing API: bzip2 compression.\n");
176 
177  /* Clear the data array */
178  memset(array,0,sizeof(float)*actualproduct);
179 
180  /* Create a file */
181  CHECK(nc_create(TESTFILE, NC_NETCDF4|NC_CLOBBER, &ncid));
182 
183  /* Do not use fill for this file */
184  CHECK(nc_set_fill(ncid, NC_NOFILL, NULL));
185 
186  /* Define the dimensions */
187  for(i=0;i<actualdims;i++) {
188  char dimname[1024];
189  snprintf(dimname,sizeof(dimname),"dim%d",i);
190  CHECK(nc_def_dim(ncid, dimname, dims[i], &dimids[i]));
191  }
192 
193  /* Define the variable */
194  CHECK(nc_def_var(ncid, "var", NC_FLOAT, actualdims, dimids, &varid));
195 
196  /* Set chunking on the variable */
197  CHECK(nc_def_var_chunking(ncid,varid,NC_CHUNKED,chunks));
198 
199  /* Verify that chunking succeeded */
200  if(!verifychunks())
201  return NC_EINVAL;
202  /* Set bzip2 compression for the variable: takes one parameter == level */
203  CHECK(nc_def_var_filter(ncid,varid,BZIP2_ID,1,&level));
204 
205  /* Read back the compression info and verify it */
206  level = 0;
207  CHECK(nc_inq_var_filter_info(ncid,varid,BZIP2_ID,&nparams,&level));
208  if(nparams != 1 || level != BZIP2_LEVEL) {
209  printf("test_filter: filter def/inq mismatch\n");
210  return NC_EFILTER;
211  }
212  /* Show the level */
213  printf("show parameters for bzip2: level=%u\n",level);
214  /* Show chunking */
215  printf("show chunks:");
216  for(i=0;i<actualdims;i++)
217  printf("%s%ld",(i==0?" chunks=":","),(unsigned long)chunks[i]);
218  printf("\n");
219 
220  /* prepare to write */
221  CHECK(nc_enddef(ncid));
222 
223  /* Fill in the array */
224  for(i=0;i<actualproduct;i++)
225  expected[i] = (float)i;
226 
227  /* write array */
228  CHECK(nc_put_var(ncid,varid,expected));
229 
230  /* Close file */
231  CHECK(nc_close(ncid));
232 
233  /* Now re-open and verify */
234  printf("\n*** Testing API: bzip2 decompression.\n");
235 
236  /* Clear the data array */
237  memset(array,0,sizeof(float)*actualproduct);
238 
239  /* Open the file */
240  CHECK(nc_open(TESTFILE, NC_NOWRITE, &ncid));
241 
242  /* Get the variable id */
243  CHECK(nc_inq_varid(ncid, "var", &varid));
244 
245  /* Check the compression algorithm */
246  filterid = 0;
247  nparams = 0;
248  params = NULL;
249  CHECK(nc_inq_var_filter(ncid,varid,&filterid,&nparams,NULL));
250  if(filterid == 0) CHECK(NC_ENOFILTER); /* Not defined */
251  if(nparams > 0) {
252  params = (unsigned int*)malloc(sizeof(unsigned int)*nparams);
253  if(params == NULL)
254  return NC_ENOMEM;
255  CHECK(nc_inq_var_filter(ncid,varid,&filterid,&nparams,params));
256  }
257  if(filterid != BZIP2_ID) {
258  printf("Bzip2 id mismatch: %d\n",filterid);
259  return NC_EFILTER;
260  }
261  if(nparams != 1 && params != NULL && params[0] != BZIP2_LEVEL) {
262  printf("Compression parameter mismatch\n");
263  return NC_EFILTER;
264  }
265 
266  /* Verify chunking */
267  if(!verifychunks())
268  return 0;
269 
270  /* Read the data */
271  CHECK(nc_get_var_float(ncid, varid, array));
272 
273  /* Close the file */
274  CHECK(nc_close(ncid));
275  return (compare() == NC_NOERR ? 0 : 1);
276 }
277 
278 /**************************************************/
279 /* Utilities */
280 
281 static void
282 init(int argc, char** argv)
283 {
284  int i;
285  /* Setup various variables */
286  actualproduct = 1;
287  chunkproduct = 1;
288  for(i=0;i<NDIMS;i++) {
289  dims[i] = dimsize;
290  chunks[i] = chunksize;
291  if(i < actualdims) {
292  actualproduct *= dims[i];
293  chunkproduct *= chunks[i];
294  }
295  }
296  /* Allocate max size */
297  array = (float*)calloc(1,sizeof(float)*actualproduct);
298  expected = (float*)calloc(1,sizeof(float)*actualproduct);
299 }
300 
301 /**************************************************/
302 int
303 main(int argc, char **argv)
304 {
305  H5Eprint1(stderr);
306  init(argc,argv);
307  if(test_bzip2() != NC_NOERR) ERRR;
308  exit(nerrs > 0?1:0);
309 }
310 
NC_NOERR
#define NC_NOERR
No Error.
Definition: netcdf.h:333
NC_EINVAL
#define NC_EINVAL
Invalid Argument.
Definition: netcdf.h:343
NC_NOWRITE
#define NC_NOWRITE
Set read-only access for nc_open().
Definition: netcdf.h:125
NC_FLOAT
#define NC_FLOAT
single precision floating point number
Definition: netcdf.h:40
NC_ESTORAGE
#define NC_ESTORAGE
Can't specify both contiguous and chunking.
Definition: netcdf.h:471
NC_NOFILL
#define NC_NOFILL
Argument to nc_set_fill() to turn off filling of data.
Definition: netcdf.h:115
NC_ENOFILTER
#define NC_ENOFILTER
Filter not defined on variable.
Definition: netcdf.h:482
nc_def_var
EXTERNL int nc_def_var(int ncid, const char *name, nc_type xtype, int ndims, const int *dimidsp, int *varidp)
Define a new variable.
Definition: dvar.c:212
nc_def_dim
EXTERNL int nc_def_dim(int ncid, const char *name, size_t len, int *idp)
Define a new dimension.
Definition: ddim.c:121
nc_def_var_chunking
EXTERNL int nc_def_var_chunking(int ncid, int varid, int storage, const size_t *chunksizesp)
Define storage and, if chunked storage is used, chunking parameters for a variable.
Definition: dvar.c:622
NC_EFILTER
#define NC_EFILTER
Filter operation failed.
Definition: netcdf.h:478
NC_CHUNKED
#define NC_CHUNKED
In HDF5 files you can set storage for each variable to be either contiguous or chunked,...
Definition: netcdf.h:299
NC_EBADCHUNK
#define NC_EBADCHUNK
Bad chunksize.
Definition: netcdf.h:472
nc_put_var
EXTERNL int nc_put_var(int ncid, int varid, const void *op)
Definition: dvarput.c:920
nc_close
EXTERNL int nc_close(int ncid)
Definition: dfile.c:1291
nc_get_var_float
int nc_get_var_float(int ncid, int varid, float *ip)
Definition: dvarget.c:1075
netcdf.h
nc_inq_varid
EXTERNL int nc_inq_varid(int ncid, const char *name, int *varidp)
Definition: dvarinq.c:60
nc_set_fill
EXTERNL int nc_set_fill(int ncid, int fillmode, int *old_modep)
Definition: dfile.c:1465
nc_open
EXTERNL int nc_open(const char *path, int mode, int *ncidp)
Open an existing netCDF file.
Definition: dfile.c:655
NC_ENOMEM
#define NC_ENOMEM
Memory allocation (malloc) failure.
Definition: netcdf.h:413
nc_strerror
EXTERNL const char * nc_strerror(int ncerr)
Given an error number, return an error message.
Definition: derror.c:87
NC_NETCDF4
#define NC_NETCDF4
Use netCDF-4/HDF5 format.
Definition: netcdf.h:152
nc_enddef
EXTERNL int nc_enddef(int ncid)
Definition: dfile.c:1018
nc_create
EXTERNL int nc_create(const char *path, int cmode, int *ncidp)
Definition: dfile.c:389
NC_EBADDIM
#define NC_EBADDIM
Invalid dimension id or name.
Definition: netcdf.h:376
nc_inq_var_chunking
EXTERNL int nc_inq_var_chunking(int ncid, int varid, int *storagep, size_t *chunksizesp)
Definition: dvarinq.c:466
NC_CLOBBER
#define NC_CLOBBER
Destroy existing file.
Definition: netcdf.h:128