Bug Summary

File:libraries/DAQ/HDEVIO.cc
Location:line 100, column 2
Description:Value stored to 'left' is never read

Annotated Source Code

1// $Id$
2//
3// File: HDEVIO.cc
4// Created: Wed Dec 10 07:22:00 EST 2014
5// Creator: davidl (on Darwin harriet 13.4.0 i386)
6//
7
8#include <stdlib.h>
9#include <string.h>
10
11#include "HDEVIO.h"
12
13//---------------------------------
14// HDEVIO (Constructor)
15//---------------------------------
16HDEVIO::HDEVIO(string filename):filename(filename)
17{
18 // These must be initialized in case we return early
19 // so they aren't deleted in the destructor if they
20 // were never allocated.
21 fbuff = NULL__null;
22 buff = NULL__null;
23
24 is_open = false;
25 ifs.open(filename.c_str());
26 if(!ifs.is_open()){
27 ClearErrorMessage();
28 err_mess << "Unable to open EVIO file: " << filename;
29 return;
30 }
31
32 fbuff_size = 10000000; // 40MB input buffer
33 fbuff = new uint32_t[fbuff_size];;
34 fnext = fbuff;
35 fbuff_end = fbuff;
36 fbuff_len = 0;
37 _gcount = 0;
38
39 buff_limit = 5000000; // Don't allow us to allocate more than 5M words for read buffer
40 buff_size = 1024; // initialize with 4kB buffer
41 buff_len = 0;
42 buff = new uint32_t[buff_size];
43 next = buff; // needed so initial calculation of left is 0
44 last_event_pos = 0;
45 last_event_len = 0;
46 err_code = HDEVIO_OK;
47
48 Nblocks = 0;
49 Nevents = 0;
50 Nerrors = 0;
51 Nbad_blocks = 0;
52 Nbad_events = 0;
53
54 event_type_mask = 0xFFFF; // default to accepting all types
55 is_mapped = false;
56
57 NB_next_pos = 0;
58
59 ifs.seekg(0, ios_base::end);
60 total_size_bytes = ifs.tellg();
61 ifs.seekg(0, ios_base::beg);
62
63 is_open = true;
64}
65
66//---------------------------------
67// ~HDEVIO (Destructor)
68//---------------------------------
69HDEVIO::~HDEVIO()
70{
71 if(ifs.is_open()) ifs.close();
72 if(buff ) delete[] buff;
73 if(fbuff) delete[] fbuff;
74}
75
76//---------------------------------
77// buff_read
78//---------------------------------
79void HDEVIO::buff_read(char* s, streamsize nwords)
80{
81 /// Read n bytes into the specified buffer.
82 /// This will read from the file if needed, or simply copy
83 /// the bytes from the existing fbuff. It is purposely meant
84 /// as a drop-in replacement for ifstream::read so that we
85 /// can make actual reads from the file in larger chunks and
86 /// avoid backwards seeks on the actual file. This is needed
87 /// because the Lustre filesystem is optimized for large data
88 /// reads but smaller reads with backwards seeks tend to cripple
89 /// its performance. The main difference between this and
90 /// ifstream::read is that this does not return an istream&
91 /// reference.
92
93 // Number of words left in fbuff
94 uint64_t left = ((uint64_t)fbuff_end - (uint64_t)fnext)/sizeof(uint32_t);
95
96 // First, copy what's already in fbuff
97 uint64_t Ncopied = nwords<(int64_t)left ? (uint64_t)nwords:left;
98 _gcount = Ncopied*sizeof(uint32_t);
99 if(_gcount>0) memcpy((char*)s, (char*)fnext, _gcount);
100 left -= Ncopied;
Value stored to 'left' is never read
101 fnext += Ncopied;
102 s += _gcount; // advance pointer to user buff in case we need to write more
103
104 // If needed, read in another chunk from the file.
105 // Try and keep last 8 words so if a seekg is called to back
106 // us up that amount, we don't have to call seekg on the file.
107 if( (int64_t)Ncopied < nwords ){
108
109 // Initialize to start of buffer
110 uint32_t *myfbuff = fbuff;
111 uint32_t myfbuff_size = fbuff_size;
112 fbuff_len = 0;
113
114 // If at least 8 words exist in fbuff, then copy last
115 // 8 words into front of fbuff.
116 if( fbuff_len >= 8 ){
117 memcpy((char*)fbuff, (char*)&fbuff[fbuff_len-8], 8*sizeof(uint32_t));
118 myfbuff = &fbuff[8];
119 myfbuff_size -= 8;
120 fbuff_len += 8;
121 }
122
123 // Read in chunk from file
124 ifs.read((char*)myfbuff, myfbuff_size*sizeof(uint32_t));
125 fbuff_len += (uint64_t)(ifs.gcount()/sizeof(uint32_t));
126 fnext = myfbuff;
127 fbuff_end = &fbuff[fbuff_len];
128
129 // Copy remainder of request
130 uint64_t myleft = ((uint64_t)fbuff_end - (uint64_t)fnext)/sizeof(uint32_t);
131
132 uint64_t mynwords = nwords - Ncopied;
133 uint64_t myNcopied = mynwords<myleft ? mynwords:myleft;
134 uint64_t mygcount = myNcopied*sizeof(uint32_t);
135 if(mygcount>0) memcpy((char*)s, (char*)fnext, mygcount);
136 fnext += myNcopied;
137 _gcount += myNcopied*sizeof(uint32_t);
138 }
139}
140
141//---------------------------------
142// buff_seekg
143//---------------------------------
144void HDEVIO::buff_seekg (streamoff off, ios_base::seekdir way)
145{
146 // Convert offset from bytes to words
147 int64_t off_words = (int64_t)off/(int64_t)sizeof(uint32_t);
148
149 // find current position relative to start of buffer in words
150 int64_t fpos = (int64_t)((uint64_t)fnext - (uint64_t)fbuff)/sizeof(uint32_t);
151
152 // Seek depending on what user has requested
153 if( way == ios_base::cur ){
154
155 // Add requested offset
156 fpos += off_words; // desired position relative to start of fbuff
157
158 if(fpos>=0 && fpos<(int64_t)fbuff_len){
159 // Request is to a point inside current buffer
160 fnext = &fbuff[fpos];
161 _gcount = 0;
162 }else if(fpos<0){
163 // Seek point is outside of buffer. Move file pointer
164 // and indicate buffer is now empty so next buff_read()
165 // call will force a read.
166
167 // Current file position should be just after end of fbuff
168 // Subtract fbuff_len to move it back to start of fbuff.
169 // fpos is position relative to start of fbuff.
170 off = (streamoff)fpos - (streamoff)fbuff_len; // offset relative to actual current file position
171
172 ifs.seekg(off);
173
174 // Set fbuff parameters to indicate no valid data
175 fnext = fbuff;
176 fbuff_end = fbuff;
177 fbuff_len = 0;
178 _gcount = 0;
179
180 }
181
182 }else{
183 _DBG_cout<<"libraries/DAQ/HDEVIO.cc"<<":"<<183<<
" "<<"buff_seekg called with something other than ios_base::cur is unsupported!" << endl;
184 exit(-1);
185 }
186}
187
188//---------------------------------
189// ReadBlock
190//---------------------------------
191bool HDEVIO::ReadBlock(void)
192{
193 /// Read in the next EVIO block. Return true if successful
194 /// and false otherwise.
195
196 err_code = HDEVIO_OK;
197 next = NULL__null;
198
199 buff_read((char*)buff, 8);
200 uint32_t valid_words = buff_gcount()/sizeof(uint32_t);
201 if(valid_words != 8){
202 SetErrorMessage("Could not read in 8 word EVIO block header!");
203 err_code = HDEVIO_FILE_TRUNCATED;
204 Nerrors++;
205 return false;
206 }
207
208 // Check endianess
209 if(buff[7]!=0xc0da0100 && buff[7]!=0x0001dac0){
210 ClearErrorMessage();
211 err_mess << "Magic word not valid!: " << HexStr(buff[7]) << endl;
212 err_code = HDEVIO_BAD_BLOCK_HEADER;
213 Nerrors++;
214 Nbad_blocks++;
215 return false;
216 }
217 swap_needed = (buff[7]==0x0001dac0);
218
219 // Swap header (if needed)
220 if(swap_needed) swap_block(buff, 8, buff);
221
222 // Re-allocate buffer if needed so we can read in entire block
223 uint32_t block_length = buff[0];
224 if(buff_size < block_length){
225 if(block_length > buff_limit){
226 ClearErrorMessage();
227 err_mess << "ERROR: EVIO block length greater than allocation limit (" << block_length <<" > " << block_length << " words)" << endl;
228 err_code = HDEVIO_BLOCKSIZE_GREATER_THAN_LIMIT;
229 Nerrors++;
230 return false;
231 }
232 if(buff) delete[] buff;
233 buff_size = block_length;
234 buff = new uint32_t[buff_size];
235 if(buff == NULL__null){
236 ClearErrorMessage();
237 err_mess << "ERROR: unable to allocate " << block_length <<" words" << endl;
238 err_code = HDEVIO_MEMORY_ALLOCATION_ERROR;
239 return false;
240 }
241
242 // Re-read in the block header
243 buff_seekg(-8*sizeof(uint32_t), ifs.cur);
244 buff_read((char*)buff, 8);
245 if(swap_needed) swap_block(buff, 8, buff);
246 }
247
248 if(block_length == 8){
249 // block_length =8 indicates end of file.
250 SetErrorMessage("end of file");
251 err_code = HDEVIO_EOF;
252 return false;
253 }
254
255 // Read payload of block
256 buff_read((char*)&buff[8], (block_length-8));
257 valid_words = 8 + buff_gcount()/sizeof(uint32_t);
258 if(valid_words < block_length){
259 ClearErrorMessage();
260 err_mess << "Error reading in EVIO entire block! (block number: " << buff[1] << ")" << endl;
261 err_mess << "valid_words="<<valid_words << " block_length=" << block_length;
262 err_code = HDEVIO_FILE_TRUNCATED;
263 Nerrors++;
264 return false;
265 }
266
267 // Set pointers
268 buff_len = valid_words;
269 buff_end = &buff[valid_words];
270 next = &buff[8];
271 bh = (BLOCKHEADER_t*)buff;
272
273 Nblocks++;
274 return true;
275}
276
277//---------------------------------
278// read
279//---------------------------------
280bool HDEVIO::read(uint32_t *user_buff, uint32_t user_buff_len, bool allow_swap)
281{
282 /// Read the next EVIO event into the user supplied buffer.
283 /// Return true if successful and false otherwise. Details of
284 /// the error will be in err_mess.
285
286 // If only certain event types are requested then
287 // defer to the sparse reader
288 if(event_type_mask != 0xFFFF){
289 return readSparse(user_buff, user_buff_len, allow_swap);
290 }
291
292 err_code = HDEVIO_OK;
293
294 // calculate remaining valid words in buffer
295 uint32_t left = buff_len - (uint32_t)(((unsigned long)next - (unsigned long)buff)/sizeof(uint32_t));
296
297 // Read in another event block if necessary
298 if(left < 1 || next==NULL__null){
299 bool isgood = ReadBlock();
300 if(!isgood) return false;
301 left = buff_len - 8;
302 }
303
304 if(next == NULL__null){
305 SetErrorMessage("No valid events in buffer");
306 err_code = HDEVIO_NO_EVENTS_IN_BUFFER;
307 return false;
308 }
309
310 // Check if next event will fit into user supplied buffer
311 uint32_t event_len = next[0];
312 if(swap_needed) swap_block(&event_len, 1, &event_len);
313 event_len++; // include length word for EVIO bank
314 last_event_len = event_len;
315
316 // Check that event isn't claiming to be larger than EVIO block
317 if( event_len > left ){
318 ClearErrorMessage();
319 err_mess << "WARNING: EVIO bank indicates a bigger size than block header (" << event_len << " > " << left << ")";
320 next = &buff[buff_len]; // setup so subsequent call will read in another block
321 err_code = HDEVIO_EVENT_BIGGER_THAN_BLOCK;
322 Nerrors++;
323 Nbad_blocks++;
324 return false;
325 }
326
327 // Check if user buffer is big enough to hold this
328 if(event_len > user_buff_len){
329 ClearErrorMessage();
330 err_mess << "user buffer too small for event (" << user_buff_len << " < " << event_len << ")";
331 err_code = HDEVIO_USER_BUFFER_TOO_SMALL;
332 return false;
333 }
334
335 // Copy entire event into user buffer, swapping if needed during copy
336 bool isgood = true;
337 if(swap_needed && allow_swap){
338 uint32_t Nswapped = swap_bank(user_buff, next, event_len);
339 isgood = (Nswapped == event_len);
340 }else{
341 memcpy(user_buff, next, event_len*sizeof(uint32_t));
342 }
343
344 // Advance next pointer to next EVIO event or past end of buffer.
345 next = &next[event_len];
346
347 if(isgood) Nevents++;
348
349 return isgood;
350}
351
352//---------------------------------
353// readSparse
354//---------------------------------
355bool HDEVIO::readSparse(uint32_t *user_buff, uint32_t user_buff_len, bool allow_swap)
356{
357 /// This is an alternative to the read(...) method above that
358 /// is used when the user has specified that only certain
359 /// event types are desired. This really only makes sense
360 /// for EPICS and SYNC events. This method does not use the
361 /// fbuff system and instead reads directly from the file
362 /// after seeking to the desired location determined from
363 /// a previously generated map.
364
365 err_code = HDEVIO_OK;
366 ClearErrorMessage();
367
368 // Make sure we've mapped this file
369 if(!is_mapped) MapBlocks();
370
371 // Loop over all events of all blocks looking for the next
372 // event matching the currently set type mask.
373 for(; sparse_block_iter!=evio_blocks.end(); sparse_block_iter++, sparse_event_idx = 0){
374
375 // Filter out blocks of the wrong type
376 EVIOBlockRecord &br = *sparse_block_iter;
377 // uint32_t type = (1 << br.block_type);
378
379 for(; sparse_event_idx < br.evio_events.size(); sparse_event_idx++){
380 EVIOEventRecord &er = sparse_block_iter->evio_events[sparse_event_idx];
381
382 uint32_t etype = (1 << er.event_type);
383 if( etype & event_type_mask ) break;
384 }
385 if(sparse_event_idx >= br.evio_events.size()) continue;
386
387 EVIOEventRecord &er = sparse_block_iter->evio_events[sparse_event_idx];
388
389 uint32_t event_len = er.event_len;
390 last_event_len = event_len;
391
392 // Check if user buffer is big enough to hold block
393 if( event_len > user_buff_len ){
394 ClearErrorMessage();
395 err_mess << "user buffer too small for event (" << user_buff_len << " < " << event_len << ")";
396 err_code = HDEVIO_USER_BUFFER_TOO_SMALL;
397 return false;
398 }
399
400 // At this point we're committed to reading this event so go
401 // ahead and increment pointer to next event so no matter
402 // what happens below, we don't try reading it again.
403 sparse_event_idx++;
404
405 // Set file pointer to start of EVIO event (NOT block header!)
406 last_event_pos = er.pos;
407 ifs.seekg(last_event_pos, ios_base::beg);
408
409 // Read data directly into user buffer
410 ifs.read((char*)user_buff, event_len*sizeof(uint32_t));
411
412 // Swap entire bank if needed
413 swap_needed = br.swap_needed; // set flag in HDEVIO
414 bool isgood = true;
415 if(br.swap_needed && allow_swap){
416 uint32_t Nswapped = swap_bank(user_buff, user_buff, event_len);
417 isgood = (Nswapped == event_len);
418 }
419
420 // Double check that event length matches EVIO block header
421 // but only if we either don't need to swap or need to and
422 // were allowed to (otherwise, the test will almost certainly
423 // fail!)
424 if( (!br.swap_needed) || (br.swap_needed && allow_swap) ){
425 if( (user_buff[0]+1) != event_len ){
426 ClearErrorMessage();
427 err_mess << "WARNING: EVIO bank indicates a different size than block header (" << event_len << " != " << (user_buff[0]+1) << ")";
428 err_code = HDEVIO_EVENT_BIGGER_THAN_BLOCK;
429 Nerrors++;
430 Nbad_blocks++;
431 return false;
432 }
433 }
434
435 if(isgood) Nevents++;
436
437 return isgood;
438 }
439
440 // If we got here then we did not find an event of interest
441 // above. Report that there are no more events in the file.
442 SetErrorMessage("No more events");
443 err_code = HDEVIO_EOF;
444 return false; // isgood=false
445}
446
447//---------------------------------
448// readNoFileBuff
449//---------------------------------
450bool HDEVIO::readNoFileBuff(uint32_t *user_buff, uint32_t user_buff_len, bool allow_swap)
451{
452 /// This is an alternative to the read(...) method above that
453 /// does not use a large primary file buffer. A single EVIO
454 /// header is read in at a time and the events within the block
455 /// mapped just like when using readSparse. The difference is
456 /// that here, only a single block is mapped at a time rather
457 /// than trying to map the entire file before starting. This
458 /// gives a faster start up. This may be quicker for most
459 /// desktop filesystems but may be slower for Lustre file systems
460 /// that are configured for large volume data transfers and show
461 /// perfomance degredation with small reads.
462
463 err_code = HDEVIO_OK;
464 ClearErrorMessage();
465
466 // Check if we need to read in a block header using the
467 // current file position
468 EVIOBlockRecord &br = NB_block_record;
469 if(br.evio_events.empty()){
470
471 // Check if we are at end of file
472 uint64_t words_left_in_file = (total_size_bytes-NB_next_pos)/4;
473 if( words_left_in_file == 8 ){ // (if <8 then let read below fail and return HDEVIO_FILE_TRUNCATED)
474 SetErrorMessage("No more events");
475 err_code = HDEVIO_EOF;
476 return false;
477 }
478
479 // read EVIO block header
480 BLOCKHEADER_t bh;
481 ifs.seekg( NB_next_pos, ios_base::beg);
482 ifs.clear();
483 ifs.read((char*)&bh, sizeof(bh));
484 if(!ifs.good()){
485 err_mess << "Error reading EVIO block header (truncated?)";
486 err_code = HDEVIO_FILE_TRUNCATED;
487 return false;
488 }
489
490 // Check if we need to byte swap and simultaneously
491 // verify header is good by checking magic word
492 bool swap_needed = false;
493 if(bh.magic==0x0001dac0){
494 swap_needed = true;
495 }else{
496 if(bh.magic!=0xc0da0100){
497 err_mess.str("Bad magic word");
498 err_code = HDEVIO_BAD_BLOCK_HEADER;
499 return false;
500 }
501 }
502
503 if(swap_needed)swap_block((uint32_t*)&bh, sizeof(bh)>>2, (uint32_t*)&bh);
504
505 Nblocks++;
506 streampos pos = ifs.tellg() - (streampos)sizeof(bh);
507
508 if( (uint64_t)(pos+(streampos)bh.length) > total_size_bytes ){
509 err_mess << "EVIO block extends past end of file!";
510 err_code = HDEVIO_FILE_TRUNCATED;
511 return false;
512 }
513
514 br.pos = pos;
515 br.block_len = bh.length;
516 br.swap_needed = swap_needed;
517 br.first_event = 0;
518 br.last_event = 0;
519
520 MapEvents(bh, br);
521
522 NB_next_pos = pos + (streampos)(bh.length<<2);
523 }
524
525 // Check if we did not find an event of interest above.
526 // If not, report that there are no more events in the file.
527 if(br.evio_events.empty() || !ifs.good()){
528 SetErrorMessage("No more events");
529 err_code = HDEVIO_EOF;
530 return false; // isgood=false
531 }
532
533 // Grab next event record
534 EVIOEventRecord &er = br.evio_events.front();
535
536 uint32_t event_len = er.event_len;
537 last_event_len = event_len;
538
539 // Check if user buffer is big enough to hold block
540 if( event_len > user_buff_len ){
541 ClearErrorMessage();
542 err_mess << "user buffer too small for event (" << user_buff_len << " < " << event_len << ")";
543 err_code = HDEVIO_USER_BUFFER_TOO_SMALL;
544 return false;
545 }
546
547 // Set file pointer to start of EVIO event (NOT block header!)
548 last_event_pos = er.pos;
549 ifs.seekg(last_event_pos, ios_base::beg);
550
551 // Read data directly into user buffer
552 ifs.read((char*)user_buff, event_len*sizeof(uint32_t));
553 if(!ifs.good()){
554 SetErrorMessage("No more events");
555 err_code = HDEVIO_EOF;
556 return false; // isgood=false
557 }
558
559 // Remove EVIO Event record, effectively advancing to
560 // next event for the next time we're called
561 br.evio_events.erase(NB_block_record.evio_events.begin());
562
563 // Swap entire bank if needed
564 swap_needed = br.swap_needed; // set flag in HDEVIO
565 bool isgood = true;
566 if(br.swap_needed && allow_swap){
567 uint32_t Nswapped = swap_bank(user_buff, user_buff, event_len);
568 isgood = (Nswapped == event_len);
569 }
570
571 // Double check that event length matches EVIO block header
572 // but only if we either don't need to swap or need to and
573 // were allowed to (otherwise, the test will almost certainly
574 // fail!)
575 if( (!br.swap_needed) || (br.swap_needed && allow_swap) ){
576 if( (user_buff[0]+1) != event_len ){
577 ClearErrorMessage();
578 err_mess << "WARNING: EVIO bank indicates a different size than block header (" << event_len << " != " << (user_buff[0]+1) << ")";
579 err_code = HDEVIO_EVENT_BIGGER_THAN_BLOCK;
580 Nerrors++;
581 Nbad_blocks++;
582 return false;
583 }
584 }
585
586 if(isgood) Nevents++;
587
588 return isgood;
589}
590
591//------------------------
592// rewind
593//------------------------
594void HDEVIO::rewind(void)
595{
596 /// This can be used whe reading from a file to
597 /// reset the file pointer and other position holders
598 /// to the begining of the file. This is done when
599 /// the "LOOP_FOREVER" option is used in the event source
600 /// to continuously re-read a file, essesntially making
601 /// it an infinite stream of events that can be used for
602 /// testing.
603
604 ifs.seekg(0, ios_base::beg);
605 ifs.clear();
606
607 sparse_block_iter = evio_blocks.begin();
608 sparse_event_idx = 0;
609
610 NB_block_record.evio_events.clear();
611 NB_next_pos = 0;
612
613 ClearErrorMessage();
614 err_code = HDEVIO_OK;
615}
616
617//------------------------
618// SetEventMask
619//------------------------
620uint32_t HDEVIO::SetEventMask(uint32_t mask)
621{
622 uint32_t prev_mask = event_type_mask;
623 event_type_mask = mask;
624
625 return prev_mask;
626}
627
628//------------------------
629// SetEventMask
630//------------------------
631uint32_t HDEVIO::SetEventMask(string types_str)
632{
633 uint32_t prev_mask = event_type_mask;
634
635 event_type_mask = 0;
636 if(types_str.find("BOR" ) != string::npos) event_type_mask |= (1<<kBT_BOR);
637 if(types_str.find("EPICS" ) != string::npos) event_type_mask |= (1<<kBT_EPICS);
638 if(types_str.find("PHYSICS") != string::npos) event_type_mask |= (1<<kBT_PHYSICS);
639
640 return prev_mask;
641}
642
643//------------------------
644// AddToEventMask
645//------------------------
646uint32_t HDEVIO::AddToEventMask(string type_str)
647{
648 return 0;
649}
650
651//------------------------
652// GetEVIOBlockRecords
653//------------------------
654vector<HDEVIO::EVIOBlockRecord>& HDEVIO::GetEVIOBlockRecords(void)
655{
656 if(!is_mapped) MapBlocks();
657
658 return evio_blocks;
659}
660
661//------------------------
662// MapBlocks
663//------------------------
664void HDEVIO::MapBlocks(bool print_ticker)
665{
666 if(!is_open){
667 err_mess.str("File is not open");
668 err_code = HDEVIO_FILE_NOT_OPEN;
669 return;
670 }
671
672 // Remember current file pos so we can restore it.
673 streampos start_pos = ifs.tellg();
674
675 if(print_ticker) cout << "Mapping EVIO file ..." << endl;
676
677 // Rewind to beginning of file and loop over all blocks
678 ifs.seekg(0, ios_base::beg);
679 BLOCKHEADER_t bh;
680 uint64_t Nblocks = 0;
681 while(ifs.good()){
682 ifs.read((char*)&bh, sizeof(bh));
683 if(!ifs.good()) break;
684
685 // Check if we need to byte swap and simultaneously
686 // verify header is good by checking magic word
687 bool swap_needed = false;
688 if(bh.magic==0x0001dac0){
689 swap_needed = true;
690 }else{
691 if(bh.magic!=0xc0da0100){
692 err_mess.str("Bad magic word");
693 err_code = HDEVIO_BAD_BLOCK_HEADER;
694 break;
695 }
696 }
697
698 if(swap_needed)swap_block((uint32_t*)&bh, sizeof(bh)>>2, (uint32_t*)&bh);
699
700 Nblocks++;
701 streampos block_len_bytes = (bh.length<<2)-sizeof(bh); // <<2 is for *4
702 streampos pos = ifs.tellg() - (streampos)sizeof(bh);
703
704 EVIOBlockRecord br;
705 br.pos = pos;
706 br.block_len = bh.length;
707 br.swap_needed = swap_needed;
708 br.first_event = 0;
709 br.last_event = 0;
710
711 // Categorize this block
712 uint32_t tag = bh.header>>16;
713 uint32_t M = bh.header&0xFF;
714
715 switch(tag){
716 case 0xFFD0: br.block_type = kBT_SYNC; break;
717 case 0xFFD1: br.block_type = kBT_PRESTART; break;
718 case 0xFFD2: br.block_type = kBT_GO; break;
719 case 0xFFD3: br.block_type = kBT_PAUSE; break;
720 case 0xFFD4: br.block_type = kBT_END; break;
721 case 0x0060: br.block_type = kBT_EPICS; break;
722 case 0x0070: br.block_type = kBT_BOR; break;
723 case 0xFF50:
724 case 0xFF51:
725 case 0xFF70:
726 br.block_type = kBT_PHYSICS;
727 br.first_event = bh.physics.first_event_lo;
728 br.first_event += ((uint64_t)bh.physics.first_event_hi)<<32;
729 br.last_event = br.first_event + (uint64_t)M - 1;
730 break;
731 default:
732 _DBG_cout<<"libraries/DAQ/HDEVIO.cc"<<":"<<732<<
" " << "Uknown tag: " << hex << tag << dec << endl;
733 }
734
735 // Scan through and map all events within this block
736 MapEvents(bh, br);
737
738 // Add block to list
739 evio_blocks.push_back(br);
740
741 // Update ticker
742 if(print_ticker){
743 if((Nblocks%500) == 0){
744 uint64_t total_MB = total_size_bytes>>20;
745 uint64_t read_MB = ifs.tellg()>>20;
746 if(Nblocks==0) cout << endl;
747 cout << Nblocks << " blocks scanned (" << read_MB << "/" << total_MB << " MB " << (100*read_MB/total_MB) << "%) \r";
748 cout.flush();
749 }
750 }
751
752 // Advance file pointer to start of next EVIO block header
753 ifs.seekg(block_len_bytes, ios_base::cur);
754 }
755
756 if(print_ticker) cout << endl;
757
758 // Setup iterators for sparse reading
759 sparse_block_iter = evio_blocks.begin();
760 sparse_event_idx = 0;
761
762 // Restore file pos and set flag that file has been mapped
763 ifs.clear();
764 ifs.seekg(start_pos, ios_base::beg);
765 is_mapped = true;
766}
767
768//---------------------------------
769// MapEvents
770//---------------------------------
771void HDEVIO::MapEvents(BLOCKHEADER_t &bh, EVIOBlockRecord &br)
772{
773 /// This is called if the EVIO block header indicates that
774 /// it contains more than one top-level event. The position
775 /// and event length of the first top-level event are passed
776 /// in as starting parameters.
777
778 // Record stream position upon entry so we can restore it at end
779 streampos start_pos = ifs.tellg();
780
781 // Calculate stream position of EVIO event
782 streampos pos = start_pos -(streampos)sizeof(BLOCKHEADER_t) + (streampos)(8<<2); // (8<<2) is 8 word EVIO block header times 4bytes/word
783 ifs.seekg(pos, ios_base::beg);
784
785 EVENTHEADER_t myeh;
786 for(uint32_t i=0; i<bh.eventcnt; i++){
787
788 // For the first iteration through this loop,
789 // we use the event header that has already been
790 // read in as part of the block header.
791 EVENTHEADER_t *eh = (EVENTHEADER_t*)&bh.event_len;
792 if(i!=0){
793 // Read in first few words of event
794 eh = &myeh;
795 ifs.read((char*)eh, sizeof(EVENTHEADER_t));
796 if(!ifs.good()) break;
797 if(br.swap_needed)swap_block((uint32_t*)eh, sizeof(EVENTHEADER_t)>>2, (uint32_t*)eh);
798 }else{
799 ifs.seekg(sizeof(EVENTHEADER_t), ios_base::cur);
800 }
801
802 if (eh->event_len < 2) {
803 // Before disabling this warning (or hiding it behind a VERBOSE flag)
804 // you should ask yourself the question, "Is this something that we
805 // should simply be ignoring, garbage bytes in the input evio file?"
806 std::cout << "HDEVIO::MapEvents warning - " << "Attempt to swap bank with len<2 (len="<<eh->event_len<<" header="<<hex<<eh->header<<dec<<" pos=" << pos << " tellg=" << ifs.tellg() << " i=" << i << ")" << std::endl;
807 Nbad_events++;
808 Nerrors++;
809 // --i; // This caused an infinite loop when reading hd_rawdata_020058_000.evio DL
810 streampos delta = (streampos)((eh->event_len+1)<<2) -
811 (streampos)sizeof(EVENTHEADER_t);
812 ifs.seekg(delta, ios_base::cur);
813 pos += (streampos)((eh->event_len+1)<<2);
814 continue;
815 }
816
817 EVIOEventRecord er;
818 er.pos = pos;
819 er.event_len = eh->event_len + 1; // +1 to include length word
820 er.event_type = kBT_UNKNOWN;
821 er.first_event = 0;
822 er.last_event = 0;
823
824 uint32_t tag = eh->header>>16;
825 uint32_t M = eh->header&0xFF;
826
827 switch(tag){
828 case 0xFFD0: er.event_type = kBT_SYNC; break;
829 case 0xFFD1: er.event_type = kBT_PRESTART; break;
830 case 0xFFD2: er.event_type = kBT_GO; break;
831 case 0xFFD3: er.event_type = kBT_PAUSE; break;
832 case 0xFFD4: er.event_type = kBT_END; break;
833 case 0x0060: er.event_type = kBT_EPICS; break;
834 case 0x0070: er.event_type = kBT_BOR; break;
835 case 0xFF50:
836 case 0xFF51:
837 case 0xFF70:
838 er.event_type = kBT_PHYSICS;
839 er.first_event = eh->physics.first_event_lo;
840 er.first_event += ((uint64_t)eh->physics.first_event_hi)<<32;
841 er.last_event = er.first_event + (uint64_t)M - 1;
842 if(er.first_event < br.first_event) br.first_event = er.first_event;
843 if(er.last_event > br.last_event ) br.last_event = er.last_event;
844 break;
845 default:
846 _DBG_cout<<"libraries/DAQ/HDEVIO.cc"<<":"<<846<<
" " << "Uknown tag: " << hex << tag << dec << endl;
847 }
848
849 br.evio_events.push_back(er);
850
851 // Move file position to start of next event
852 streampos delta = (streampos)((eh->event_len+1)<<2) - (streampos)sizeof(EVENTHEADER_t);
853 ifs.seekg(delta, ios_base::cur);
854 pos += (streampos)((eh->event_len+1)<<2);
855 }
856
857 ifs.seekg(start_pos, ios_base::beg);
858}
859
860//---------------------------------
861// swap_bank
862//---------------------------------
863uint32_t HDEVIO::swap_bank(uint32_t *outbuff, uint32_t *inbuff, uint32_t len)
864{
865 /// Swap an EVIO bank. If the bank contains data, it is automatically
866 /// swapped according to it's type. If the bank is a container of other
867 /// containers, then this repeatedly calls the swapper methods for the
868 /// appropriate container type (bank, tagsegment, segment). This means
869 /// that this method will be recursive in the cases where it is a bank
870 /// of banks.
871
872 if(len < 2){
873 SetErrorMessage("Attempt to swap bank with len<2");
874 err_code = HDEVIO_BANK_TRUNCATED;
875 Nerrors++;
876 Nbad_events++;
877 return 0;
878 }
879
880 // Swap length and header words
881 swap_block(inbuff, 2, outbuff);
882 uint32_t bank_len = outbuff[0];
883 if((bank_len+1) > len){
884 ClearErrorMessage();
885 err_mess << "WARNING: Bank length word exceeds valid words in buffer (" << bank_len+1 << " > " << len << ")";
886 err_code = HDEVIO_BANK_TRUNCATED;
887 Nerrors++;
888 Nbad_events++;
889 return 0;
890 }
891
892 uint32_t type = (outbuff[1]>>8) & 0xFF;
893 uint32_t Nwords = bank_len - 1;
894 uint32_t Nswapped = 2;
895 switch(type){
896 case 0x0a: // 64 bit unsigned int
897 case 0x08: // 64 bit double
898 case 0x09: // 64 bit signed int
899 swap_block((uint64_t*)&inbuff[2], Nwords/2, (uint64_t*)&outbuff[2]);
900 Nswapped += Nwords;
901 break;
902 case 0x01: // 32 bit unsigned int
903 case 0x02: // 32 bit float
904 case 0x0b: // 32 bit signed int
905 swap_block(&inbuff[2], Nwords, &outbuff[2]);
906 Nswapped += Nwords;
907 break;
908 case 0x05: // 16 bit unsigned int
909 case 0x04: // 16 bit signed int
910 swap_block((uint16_t*)&inbuff[2], Nwords*2, (uint16_t*)&outbuff[2]);
911 Nswapped += Nwords;
912 break;
913 case 0x00: // 32 bit unknown (not swapped)
914 case 0x07: // 8 bit unsigned int
915 case 0x06: // 8 bit signed int
916 memcpy((uint8_t*)&outbuff[2], (uint8_t*)&inbuff[2], Nwords*sizeof(uint32_t));
917 Nswapped += Nwords;
918 break;
919 case 0x0c:
920 while(Nswapped < (Nwords+2)){
921 uint32_t N = swap_tagsegment(&outbuff[Nswapped], &inbuff[Nswapped], (Nwords+2)-Nswapped);
922 if(N == 0) return Nswapped;
923 Nswapped += N;
924 }
925 break;
926 case 0x0d:
927 case 0x20:
928 while(Nswapped < (Nwords+2)){
929 uint32_t N = swap_segment(&outbuff[Nswapped], &inbuff[Nswapped], (Nwords+2)-Nswapped);
930 if(N == 0) return Nswapped;
931 Nswapped += N;
932 }
933 break;
934 case 0x0e:
935 case 0x10:
936 while(Nswapped < (Nwords+2)){
937 uint32_t N = swap_bank(&outbuff[Nswapped], &inbuff[Nswapped], (Nwords+2)-Nswapped);
938 if(N == 0) return Nswapped;
939 Nswapped += N;
940 }
941 break;
942 default:
943 ClearErrorMessage();
944 err_mess << "WARNING: unknown bank type (0x" << hex << type << dec << ")";
945 err_code = HDEVIO_UNKNOWN_BANK_TYPE;
946 Nerrors++;
947 Nbad_events++;
948 return 0;
949 break;
950 }
951
952 return Nswapped;
953}
954
955//---------------------------------
956// swap_tagsegment
957//---------------------------------
958uint32_t HDEVIO::swap_tagsegment(uint32_t *outbuff, uint32_t *inbuff, uint32_t len)
959{
960 /// Swap an EVIO tagsegment.
961
962 if(len < 1){
963 SetErrorMessage("Attempt to swap segment with len<1");
964 err_code = HDEVIO_BANK_TRUNCATED;
965 Nerrors++;
966 Nbad_events++;
967 return 0;
968 }
969
970 // Swap header/length word
971 swap_block(inbuff, 1, outbuff);
972 uint32_t bank_len = outbuff[0] & 0xFFFF;
973 if((bank_len) > len){
974 ClearErrorMessage();
975 err_mess << "Segment length word exceeds valid words in buffer (" << bank_len << " > " << len << ")";
976 err_code = HDEVIO_BANK_TRUNCATED;
977 Nerrors++;
978 Nbad_events++;
979 return 0;
980 }
981
982 uint32_t type = (outbuff[0]>>16) & 0x0F;
983 uint32_t Nwords = bank_len;
984 uint32_t Nswapped = 1;
985 switch(type){
986 case 0x0a: // 64 bit unsigned int
987 case 0x08: // 64 bit double
988 case 0x09: // 64 bit signed int
989 swap_block((uint64_t*)&inbuff[1], Nwords/2, (uint64_t*)&outbuff[1]);
990 Nswapped += Nwords;
991 break;
992 case 0x01: // 32 bit unsigned int
993 case 0x02: // 32 bit float
994 case 0x0b: // 32 bit signed int
995 swap_block(&inbuff[1], Nwords, &outbuff[1]);
996 Nswapped += Nwords;
997 break;
998 case 0x05: // 16 bit unsigned int
999 case 0x04: // 16 bit signed int
1000 swap_block((uint16_t*)&inbuff[1], Nwords*2, (uint16_t*)&outbuff[1]);
1001 Nswapped += Nwords;
1002 break;
1003 case 0x00: // 32 bit unknown (not swapped)
1004 case 0x07: // 8 bit unsigned int
1005 case 0x06: // 8 bit signed int
1006 memcpy((uint8_t*)&outbuff[1], (uint8_t*)&inbuff[1], Nwords*sizeof(uint32_t));
1007 Nswapped += Nwords;
1008 break;
1009 }
1010
1011 return Nswapped;
1012}
1013
1014//---------------------------------
1015// swap_segment
1016//---------------------------------
1017uint32_t HDEVIO::swap_segment(uint32_t *outbuff, uint32_t *inbuff, uint32_t len)
1018{
1019 /// Swap an EVIO segment.
1020
1021 if(len < 1){
1022 SetErrorMessage("Attempt to swap segment with len<1");
1023 err_code = HDEVIO_BANK_TRUNCATED;
1024 Nerrors++;
1025 Nbad_events++;
1026 return 0;
1027 }
1028
1029 // Swap header/length word
1030 swap_block(inbuff, 1, outbuff);
1031 uint32_t bank_len = outbuff[0] & 0xFFFF;
1032 if((bank_len) > len){
1033 ClearErrorMessage();
1034 err_mess << "Segment length word exceeds valid words in buffer (" << bank_len << " > " << len << ")";
1035 err_code = HDEVIO_BANK_TRUNCATED;
1036 Nerrors++;
1037 Nbad_events++;
1038 return 0;
1039 }
1040
1041 uint32_t type = (outbuff[0]>>16) & 0x3F;
1042 uint32_t Nwords = bank_len;
1043 uint32_t Nswapped = 1;
1044 switch(type){
1045 case 0x0a: // 64 bit unsigned int
1046 case 0x08: // 64 bit double
1047 case 0x09: // 64 bit signed int
1048 swap_block((uint64_t*)&inbuff[1], Nwords/2, (uint64_t*)&outbuff[1]);
1049 Nswapped += Nwords;
1050 break;
1051 case 0x01: // 32 bit unsigned int
1052 case 0x02: // 32 bit float
1053 case 0x0b: // 32 bit signed int
1054 swap_block(&inbuff[1], Nwords, &outbuff[1]);
1055 Nswapped += Nwords;
1056 break;
1057 case 0x05: // 16 bit unsigned int
1058 case 0x04: // 16 bit signed int
1059 swap_block((uint16_t*)&inbuff[1], Nwords*2, (uint16_t*)&outbuff[1]);
1060 Nswapped += Nwords;
1061 break;
1062 case 0x00: // 32 bit unknown (not swapped)
1063 case 0x07: // 8 bit unsigned int
1064 case 0x06: // 8 bit signed int
1065 memcpy((uint8_t*)&outbuff[1], (uint8_t*)&inbuff[1], Nwords*sizeof(uint32_t));
1066 Nswapped += Nwords;
1067 break;
1068 }
1069
1070 return Nswapped;
1071}
1072
1073//------------------------
1074// Print_fbuff
1075//------------------------
1076void HDEVIO::Print_fbuff(void)
1077{
1078 cout << endl;
1079 cout << " fbuff: " << hex << (uint64_t)fbuff << dec << endl;
1080 cout << " fnext: " << hex << (uint64_t)fnext << dec << endl;
1081 cout << " fbuff_end: " << hex << (uint64_t)fbuff_end << dec << endl;
1082 cout << " fbuff_size: " << fbuff_size << endl;
1083 cout << " fbuff_len: " << fbuff_len << endl;
1084 cout << " _gcount: " << _gcount << endl;
1085}
1086
1087//------------------------
1088// PrintEVIOBlockHeader
1089//------------------------
1090void HDEVIO::PrintEVIOBlockHeader(void)
1091{
1092
1093 cout << endl;
1094 cout << "EVIO Block Header:" << endl;
1095 cout << "------------------------" << endl;
1096 cout << " Block Length: " << HexStr(buff[0]) << " (" << buff[0] << " words = " << (buff[0]>>(10-2)) << " kB)" << endl;
1097 cout << " Block Number: " << HexStr(buff[1]) << endl;
1098 cout << "Header Length: " << HexStr(buff[2]) << " (should be 8)" << endl;
1099 cout << " Event Count: " << HexStr(buff[3]) << endl;
1100 cout << " Reserved 1: " << HexStr(buff[4]) << endl;
1101 cout << " Bit Info: " << HexStr(buff[5]>>8) << endl;
1102 cout << " Version: " << HexStr(buff[5]&0xFF) << endl;
1103 cout << " Reserved 3: " << HexStr(buff[6]) << endl;
1104 cout << " Magic word: " << HexStr(buff[7]) << (swap_needed ? " (after swapping)":"") << endl;
1105 cout << "Byte swapping is" << (swap_needed ? " ":" not ") << "needed" << endl;
1106}
1107
1108//------------------------
1109// PrintStats
1110//------------------------
1111void HDEVIO::PrintStats(void)
1112{
1113 cout << endl;
1114 cout << "EVIO Statistics for " << filename << " :" << endl;
1115 cout << "------------------------" << endl;
1116 cout << " Nblocks: " << Nblocks << endl;
1117 cout << " Nevents: " << Nevents << endl;
1118 cout << " Nerrors: " << Nerrors << endl;
1119 cout << "Nbad_blocks: " << Nbad_blocks << endl;
1120 cout << "Nbad_events: " << Nbad_events << endl;
1121 cout << endl;
1122}
1123
1124//------------------------
1125// PrintFileSummary
1126//------------------------
1127void HDEVIO::PrintFileSummary(void)
1128{
1129 if(!is_mapped) MapBlocks();
1130
1131 uint32_t Nsync = 0;
1132 uint32_t Nprestart = 0;
1133 uint32_t Ngo = 0;
1134 uint32_t Npause = 0;
1135 uint32_t Nend = 0;
1136 uint32_t Nepics = 0;
1137 uint32_t Nbor = 0;
1138 uint32_t Nphysics = 0;
1139
1140 uint64_t first_event = 0;
1141 uint64_t last_event = 0;
1142
1143 uint32_t map_size = evio_blocks.size()*sizeof(EVIOBlockRecord);
1144
1145 set<uint32_t> block_levels;
1146 set<uint32_t> events_in_block;
1147
1148 // Loop over all EVIO block records
1149 for(uint32_t i=0; i<evio_blocks.size(); i++){
1150 EVIOBlockRecord &br = evio_blocks[i];
1151 events_in_block.insert(br.evio_events.size());
1152 map_size += br.evio_events.size()*sizeof(EVIOEventRecord);
1153
1154 for(uint32_t j=0; j<br.evio_events.size(); j++){
1155 EVIOEventRecord &er = br.evio_events[j];
1156
1157 uint32_t block_level;
1158 switch(er.event_type){
1159 case kBT_SYNC: Nsync++; break;
1160 case kBT_PRESTART: Nprestart++; break;
1161 case kBT_GO: Ngo++; break;
1162 case kBT_PAUSE: Npause++; break;
1163 case kBT_END: Nend++; break;
1164 case kBT_EPICS: Nepics++; break;
1165 case kBT_BOR: Nbor++; break;
1166 case kBT_PHYSICS:
1167 block_level = (uint32_t)((er.last_event - er.first_event) + 1);
1168 block_levels.insert(block_level);
1169 Nphysics += block_level;
1170 if(er.first_event<first_event || first_event==0) first_event = er.first_event;
1171 if(er.last_event>last_event) last_event = er.last_event;
1172 break;
1173 default:
1174 break;
1175 }
1176
1177 //_DBG_ << "Block " << i << " event " << j << " " << er.last_event <<" - " << er.first_event << " = " << block_level << endl;
1178 }
1179 }
1180
1181 // form succint string of block levels
1182 stringstream ss;
1183 set<uint32_t>::iterator it = block_levels.begin();
1184 for(; it!=block_levels.end(); it++) ss << *it << ",";
1185 string sblock_levels = ss.str();
1186 if(!sblock_levels.empty()) sblock_levels.erase(sblock_levels.length()-1);
1187
1188 // form succint string of events per block
1189 ss.str("");
1190 it = events_in_block.begin();
1191 for(; it!=events_in_block.end(); it++){
1192 uint32_t val = *it;
1193 ss << val;
1194 if(++it==events_in_block.end()) break;
1195 if( *it == ++val ){
1196 ss << "-";
1197 for(it++; it!=events_in_block.end(); it++){
1198 if( *it != ++val ){
1199 ss << (val-1) << ",";
1200 it--;
1201 break;
1202 }
1203 }
1204 }else{
1205 ss << ",";
1206 }
1207 if( it==events_in_block.end() ) break;
1208 }
1209 string sevents_in_block = ss.str();
1210
1211 // Print results
1212 cout << endl;
1213 cout << "EVIO file size: " << (total_size_bytes>>20) << " MB" <<endl;
1214 cout << "EVIO block map size: " << (map_size>>10) << " kB" <<endl;
1215 cout << "first event: " << first_event << endl;
1216 cout << "last event: " << last_event << endl;
1217
1218 cout << endl;
1219 cout << " Nblocks = " << evio_blocks.size() << endl;
1220 cout << " block levels = " << sblock_levels << endl;
1221 cout << "events per block = " << sevents_in_block << endl;
1222 cout << " Nsync = " << Nsync << endl;
1223 cout << " Nprestart = " << Nprestart << endl;
1224 cout << " Ngo = " << Ngo << endl;
1225 cout << " Npause = " << Npause << endl;
1226 cout << " Nend = " << Nend << endl;
1227 cout << " Nepics = " << Nepics << endl;
1228 cout << " Nbor = " << Nbor << endl;
1229 cout << " Nphysics = " << Nphysics << endl;
1230 cout << endl;
1231}
1232
1233