Bug Summary

File:libraries/DAQ/HDEVIO.cc
Location:line 102, column 2
Description:Value stored to 'left' is never read

Annotated Source Code

1// $Id$
2//
3// File: HDEVIO.cc
4// Created: Wed Dec 10 07:22:00 EST 2014
5// Creator: davidl (on Darwin harriet 13.4.0 i386)
6//
7
8#include <stdlib.h>
9#include <string.h>
10
11#include "HDEVIO.h"
12
13//---------------------------------
14// HDEVIO (Constructor)
15//---------------------------------
16HDEVIO::HDEVIO(string filename):filename(filename)
17{
18 // These must be initialized in case we return early
19 // so they aren't deleted in the destructor if they
20 // were never allocated.
21 fbuff = NULL__null;
22 buff = NULL__null;
23
24 is_open = false;
25 ifs.open(filename.c_str());
26 if(!ifs.is_open()){
27 ClearErrorMessage();
28 err_mess << "Unable to open EVIO file: " << filename;
29 return;
30 }
31
32 fbuff_size = 10000000; // 40MB input buffer
33 fbuff = new uint32_t[fbuff_size];;
34 fnext = fbuff;
35 fbuff_end = fbuff;
36 fbuff_len = 0;
37 _gcount = 0;
38
39 buff_limit = 5000000; // Don't allow us to allocate more than 5M words for read buffer
40 buff_size = 1024; // initialize with 4kB buffer
41 buff_len = 0;
42 buff = new uint32_t[buff_size];
43 next = buff; // needed so initial calculation of left is 0
44 last_event_pos = 0;
45 last_event_len = 0;
46 err_code = HDEVIO_OK;
47
48 Nblocks = 0;
49 Nevents = 0;
50 Nerrors = 0;
51 Nbad_blocks = 0;
52 Nbad_events = 0;
53
54 event_type_mask = 0xFFFF; // default to accepting all types
55 is_mapped = false;
56
57 NB_next_pos = 0;
58
59 IGNORE_EMPTY_BOR = false;
60
61 ifs.seekg(0, ios_base::end);
62 total_size_bytes = ifs.tellg();
63 ifs.seekg(0, ios_base::beg);
64
65 is_open = true;
66}
67
68//---------------------------------
69// ~HDEVIO (Destructor)
70//---------------------------------
71HDEVIO::~HDEVIO()
72{
73 if(ifs.is_open()) ifs.close();
74 if(buff ) delete[] buff;
75 if(fbuff) delete[] fbuff;
76}
77
78//---------------------------------
79// buff_read
80//---------------------------------
81void HDEVIO::buff_read(char* s, streamsize nwords)
82{
83 /// Read n bytes into the specified buffer.
84 /// This will read from the file if needed, or simply copy
85 /// the bytes from the existing fbuff. It is purposely meant
86 /// as a drop-in replacement for ifstream::read so that we
87 /// can make actual reads from the file in larger chunks and
88 /// avoid backwards seeks on the actual file. This is needed
89 /// because the Lustre filesystem is optimized for large data
90 /// reads but smaller reads with backwards seeks tend to cripple
91 /// its performance. The main difference between this and
92 /// ifstream::read is that this does not return an istream&
93 /// reference.
94
95 // Number of words left in fbuff
96 uint64_t left = ((uint64_t)fbuff_end - (uint64_t)fnext)/sizeof(uint32_t);
97
98 // First, copy what's already in fbuff
99 uint64_t Ncopied = nwords<(int64_t)left ? (uint64_t)nwords:left;
100 _gcount = Ncopied*sizeof(uint32_t);
101 if(_gcount>0) memcpy((char*)s, (char*)fnext, _gcount);
102 left -= Ncopied;
Value stored to 'left' is never read
103 fnext += Ncopied;
104 s += _gcount; // advance pointer to user buff in case we need to write more
105
106 // If needed, read in another chunk from the file.
107 // Try and keep last 8 words so if a seekg is called to back
108 // us up that amount, we don't have to call seekg on the file.
109 if( (int64_t)Ncopied < nwords ){
110
111 // Initialize to start of buffer
112 uint32_t *myfbuff = fbuff;
113 uint32_t myfbuff_size = fbuff_size;
114 fbuff_len = 0;
115
116 // If at least 8 words exist in fbuff, then copy last
117 // 8 words into front of fbuff.
118 if( fbuff_len >= 8 ){
119 memcpy((char*)fbuff, (char*)&fbuff[fbuff_len-8], 8*sizeof(uint32_t));
120 myfbuff = &fbuff[8];
121 myfbuff_size -= 8;
122 fbuff_len += 8;
123 }
124
125 // Read in chunk from file
126 ifs.read((char*)myfbuff, myfbuff_size*sizeof(uint32_t));
127 fbuff_len += (uint64_t)(ifs.gcount()/sizeof(uint32_t));
128 fnext = myfbuff;
129 fbuff_end = &fbuff[fbuff_len];
130
131 // Copy remainder of request
132 uint64_t myleft = ((uint64_t)fbuff_end - (uint64_t)fnext)/sizeof(uint32_t);
133
134 uint64_t mynwords = nwords - Ncopied;
135 uint64_t myNcopied = mynwords<myleft ? mynwords:myleft;
136 uint64_t mygcount = myNcopied*sizeof(uint32_t);
137 if(mygcount>0) memcpy((char*)s, (char*)fnext, mygcount);
138 fnext += myNcopied;
139 _gcount += myNcopied*sizeof(uint32_t);
140 }
141}
142
143//---------------------------------
144// buff_seekg
145//---------------------------------
146void HDEVIO::buff_seekg (streamoff off, ios_base::seekdir way)
147{
148 // Convert offset from bytes to words
149 int64_t off_words = (int64_t)off/(int64_t)sizeof(uint32_t);
150
151 // find current position relative to start of buffer in words
152 int64_t fpos = (int64_t)((uint64_t)fnext - (uint64_t)fbuff)/sizeof(uint32_t);
153
154 // Seek depending on what user has requested
155 if( way == ios_base::cur ){
156
157 // Add requested offset
158 fpos += off_words; // desired position relative to start of fbuff
159
160 if(fpos>=0 && fpos<(int64_t)fbuff_len){
161 // Request is to a point inside current buffer
162 fnext = &fbuff[fpos];
163 _gcount = 0;
164 }else if(fpos<0){
165 // Seek point is outside of buffer. Move file pointer
166 // and indicate buffer is now empty so next buff_read()
167 // call will force a read.
168
169 // Current file position should be just after end of fbuff
170 // Subtract fbuff_len to move it back to start of fbuff.
171 // fpos is position relative to start of fbuff.
172 off = (streamoff)fpos - (streamoff)fbuff_len; // offset relative to actual current file position
173
174 ifs.seekg(off);
175
176 // Set fbuff parameters to indicate no valid data
177 fnext = fbuff;
178 fbuff_end = fbuff;
179 fbuff_len = 0;
180 _gcount = 0;
181
182 }
183
184 }else{
185 _DBG_cout<<"libraries/DAQ/HDEVIO.cc"<<":"<<185<<
" "<<"buff_seekg called with something other than ios_base::cur is unsupported!" << endl;
186 exit(-1);
187 }
188}
189
190//---------------------------------
191// ReadBlock
192//---------------------------------
193bool HDEVIO::ReadBlock(void)
194{
195 /// Read in the next EVIO block. Return true if successful
196 /// and false otherwise.
197
198 err_code = HDEVIO_OK;
199 next = NULL__null;
200
201 buff_read((char*)buff, 8);
202 uint32_t valid_words = buff_gcount()/sizeof(uint32_t);
203 if(valid_words != 8){
204 SetErrorMessage("Could not read in 8 word EVIO block header!");
205 err_code = HDEVIO_FILE_TRUNCATED;
206 Nerrors++;
207 return false;
208 }
209
210 // Check endianess
211 if(buff[7]!=0xc0da0100 && buff[7]!=0x0001dac0){
212 ClearErrorMessage();
213 err_mess << "Magic word not valid!: " << HexStr(buff[7]) << endl;
214 err_code = HDEVIO_BAD_BLOCK_HEADER;
215 Nerrors++;
216 Nbad_blocks++;
217 return false;
218 }
219 swap_needed = (buff[7]==0x0001dac0);
220
221 // Swap header (if needed)
222 if(swap_needed) swap_block(buff, 8, buff);
223
224 // Re-allocate buffer if needed so we can read in entire block
225 uint32_t block_length = buff[0];
226 if(buff_size < block_length){
227 if(block_length > buff_limit){
228 ClearErrorMessage();
229 err_mess << "ERROR: EVIO block length greater than allocation limit (" << block_length <<" > " << block_length << " words)" << endl;
230 err_code = HDEVIO_BLOCKSIZE_GREATER_THAN_LIMIT;
231 Nerrors++;
232 return false;
233 }
234 if(buff) delete[] buff;
235 buff_size = block_length;
236 buff = new uint32_t[buff_size];
237 if(buff == NULL__null){
238 ClearErrorMessage();
239 err_mess << "ERROR: unable to allocate " << block_length <<" words" << endl;
240 err_code = HDEVIO_MEMORY_ALLOCATION_ERROR;
241 return false;
242 }
243
244 // Re-read in the block header
245 buff_seekg(-8*sizeof(uint32_t), ifs.cur);
246 buff_read((char*)buff, 8);
247 if(swap_needed) swap_block(buff, 8, buff);
248 }
249
250 if(block_length == 8){
251 // block_length =8 indicates end of file.
252 SetErrorMessage("end of file");
253 err_code = HDEVIO_EOF;
254 return false;
255 }
256
257 // Read payload of block
258 buff_read((char*)&buff[8], (block_length-8));
259 valid_words = 8 + buff_gcount()/sizeof(uint32_t);
260 if(valid_words < block_length){
261 ClearErrorMessage();
262 err_mess << "Error reading in EVIO entire block! (block number: " << buff[1] << ")" << endl;
263 err_mess << "valid_words="<<valid_words << " block_length=" << block_length;
264 err_code = HDEVIO_FILE_TRUNCATED;
265 Nerrors++;
266 return false;
267 }
268
269 // Set pointers
270 buff_len = valid_words;
271 buff_end = &buff[valid_words];
272 next = &buff[8];
273 bh = (BLOCKHEADER_t*)buff;
274
275 Nblocks++;
276 return true;
277}
278
279//---------------------------------
280// read
281//---------------------------------
282bool HDEVIO::read(uint32_t *user_buff, uint32_t user_buff_len, bool allow_swap)
283{
284 /// Read the next EVIO event into the user supplied buffer.
285 /// Return true if successful and false otherwise. Details of
286 /// the error will be in err_mess.
287
288 // If only certain event types are requested then
289 // defer to the sparse reader
290 if(event_type_mask != 0xFFFF){
291 return readSparse(user_buff, user_buff_len, allow_swap);
292 }
293
294 err_code = HDEVIO_OK;
295
296 // calculate remaining valid words in buffer
297 uint32_t left = buff_len - (uint32_t)(((unsigned long)next - (unsigned long)buff)/sizeof(uint32_t));
298
299 // Read in another event block if necessary
300 if(left < 1 || next==NULL__null){
301 bool isgood = ReadBlock();
302 if(!isgood) return false;
303 left = buff_len - 8;
304 }
305
306 if(next == NULL__null){
307 SetErrorMessage("No valid events in buffer");
308 err_code = HDEVIO_NO_EVENTS_IN_BUFFER;
309 return false;
310 }
311
312 // Check if next event will fit into user supplied buffer
313 uint32_t event_len = next[0];
314 if(swap_needed) swap_block(&event_len, 1, &event_len);
315 event_len++; // include length word for EVIO bank
316 last_event_len = event_len;
317
318 // Check that event isn't claiming to be larger than EVIO block
319 if( event_len > left ){
320 ClearErrorMessage();
321 err_mess << "WARNING: EVIO bank indicates a bigger size than block header (" << event_len << " > " << left << ")";
322 next = &buff[buff_len]; // setup so subsequent call will read in another block
323 err_code = HDEVIO_EVENT_BIGGER_THAN_BLOCK;
324 Nerrors++;
325 Nbad_blocks++;
326 return false;
327 }
328
329 // Check if user buffer is big enough to hold this
330 if(event_len > user_buff_len){
331 ClearErrorMessage();
332 err_mess << "user buffer too small for event (" << user_buff_len << " < " << event_len << ")";
333 err_code = HDEVIO_USER_BUFFER_TOO_SMALL;
334 return false;
335 }
336
337 // Copy entire event into user buffer, swapping if needed during copy
338 bool isgood = true;
339 if(swap_needed && allow_swap){
340 uint32_t Nswapped = swap_bank(user_buff, next, event_len);
341 isgood = (Nswapped == event_len);
342 }else{
343 memcpy(user_buff, next, event_len*sizeof(uint32_t));
344 }
345
346 // Advance next pointer to next EVIO event or past end of buffer.
347 next = &next[event_len];
348
349 if(isgood) Nevents++;
350
351 return isgood;
352}
353
354//---------------------------------
355// readSparse
356//---------------------------------
357bool HDEVIO::readSparse(uint32_t *user_buff, uint32_t user_buff_len, bool allow_swap)
358{
359 /// This is an alternative to the read(...) method above that
360 /// is used when the user has specified that only certain
361 /// event types are desired. This really only makes sense
362 /// for EPICS and SYNC events. This method does not use the
363 /// fbuff system and instead reads directly from the file
364 /// after seeking to the desired location determined from
365 /// a previously generated map.
366
367 err_code = HDEVIO_OK;
368 ClearErrorMessage();
369
370 // Make sure we've mapped this file
371 if(!is_mapped) MapBlocks();
372
373 // Loop over all events of all blocks looking for the next
374 // event matching the currently set type mask.
375 for(; sparse_block_iter!=evio_blocks.end(); sparse_block_iter++, sparse_event_idx = 0){
376
377 // Filter out blocks of the wrong type
378 EVIOBlockRecord &br = *sparse_block_iter;
379 // uint32_t type = (1 << br.block_type);
380
381 for(; sparse_event_idx < br.evio_events.size(); sparse_event_idx++){
382 EVIOEventRecord &er = sparse_block_iter->evio_events[sparse_event_idx];
383
384 uint32_t etype = (1 << er.event_type);
385 if( etype & event_type_mask ) break;
386 }
387 if(sparse_event_idx >= br.evio_events.size()) continue;
388
389 EVIOEventRecord &er = sparse_block_iter->evio_events[sparse_event_idx];
390
391 uint32_t event_len = er.event_len;
392 last_event_len = event_len;
393
394 // Check if user buffer is big enough to hold block
395 if( event_len > user_buff_len ){
396 ClearErrorMessage();
397 err_mess << "user buffer too small for event (" << user_buff_len << " < " << event_len << ")";
398 err_code = HDEVIO_USER_BUFFER_TOO_SMALL;
399 return false;
400 }
401
402 // At this point we're committed to reading this event so go
403 // ahead and increment pointer to next event so no matter
404 // what happens below, we don't try reading it again.
405 sparse_event_idx++;
406
407 // Set file pointer to start of EVIO event (NOT block header!)
408 last_event_pos = er.pos;
409 ifs.seekg(last_event_pos, ios_base::beg);
410
411 // Read data directly into user buffer
412 ifs.read((char*)user_buff, event_len*sizeof(uint32_t));
413
414 // Swap entire bank if needed
415 swap_needed = br.swap_needed; // set flag in HDEVIO
416 bool isgood = true;
417 if(br.swap_needed && allow_swap){
418 uint32_t Nswapped = swap_bank(user_buff, user_buff, event_len);
419 isgood = (Nswapped == event_len);
420 }
421
422 // Double check that event length matches EVIO block header
423 // but only if we either don't need to swap or need to and
424 // were allowed to (otherwise, the test will almost certainly
425 // fail!)
426 if( (!br.swap_needed) || (br.swap_needed && allow_swap) ){
427 if( (user_buff[0]+1) != event_len ){
428 ClearErrorMessage();
429 err_mess << "WARNING: EVIO bank indicates a different size than block header (" << event_len << " != " << (user_buff[0]+1) << ")";
430 err_code = HDEVIO_EVENT_BIGGER_THAN_BLOCK;
431 Nerrors++;
432 Nbad_blocks++;
433 return false;
434 }
435 }
436
437 if(isgood) Nevents++;
438
439 return isgood;
440 }
441
442 // If we got here then we did not find an event of interest
443 // above. Report that there are no more events in the file.
444 SetErrorMessage("No more events");
445 err_code = HDEVIO_EOF;
446 return false; // isgood=false
447}
448
449//---------------------------------
450// readNoFileBuff
451//---------------------------------
452bool HDEVIO::readNoFileBuff(uint32_t *user_buff, uint32_t user_buff_len, bool allow_swap)
453{
454 /// This is an alternative to the read(...) method above that
455 /// does not use a large primary file buffer. A single EVIO
456 /// header is read in at a time and the events within the block
457 /// mapped just like when using readSparse. The difference is
458 /// that here, only a single block is mapped at a time rather
459 /// than trying to map the entire file before starting. This
460 /// gives a faster start up. This may be quicker for most
461 /// desktop filesystems but may be slower for Lustre file systems
462 /// that are configured for large volume data transfers and show
463 /// perfomance degredation with small reads.
464
465 err_code = HDEVIO_OK;
466 ClearErrorMessage();
467
468 // Check if we need to read in a block header using the
469 // current file position
470 EVIOBlockRecord &br = NB_block_record;
471 if(br.evio_events.empty()){
472
473 // Check if we are at end of file
474 uint64_t words_left_in_file = (total_size_bytes-NB_next_pos)/4;
475 if( words_left_in_file == 8 ){ // (if <8 then let read below fail and return HDEVIO_FILE_TRUNCATED)
476 SetErrorMessage("No more events");
477 err_code = HDEVIO_EOF;
478 return false;
479 }
480
481 // read EVIO block header
482 BLOCKHEADER_t bh;
483 ifs.seekg( NB_next_pos, ios_base::beg);
484 ifs.clear();
485 ifs.read((char*)&bh, sizeof(bh));
486 if(!ifs.good()){
487 err_mess << "Error reading EVIO block header (truncated?)";
488 err_code = HDEVIO_FILE_TRUNCATED;
489 return false;
490 }
491
492 // Check if we need to byte swap and simultaneously
493 // verify header is good by checking magic word
494 bool swap_needed = false;
495 if(bh.magic==0x0001dac0){
496 swap_needed = true;
497 }else{
498 if(bh.magic!=0xc0da0100){
499 err_mess.str("Bad magic word");
500 err_code = HDEVIO_BAD_BLOCK_HEADER;
501 return false;
502 }
503 }
504
505 if(swap_needed)swap_block((uint32_t*)&bh, sizeof(bh)>>2, (uint32_t*)&bh);
506
507 Nblocks++;
508 streampos pos = ifs.tellg() - (streampos)sizeof(bh);
509
510 if( (uint64_t)(pos+(streampos)bh.length) > total_size_bytes ){
511 err_mess << "EVIO block extends past end of file!";
512 err_code = HDEVIO_FILE_TRUNCATED;
513 return false;
514 }
515
516 br.pos = pos;
517 br.block_len = bh.length;
518 br.swap_needed = swap_needed;
519 br.first_event = 0;
520 br.last_event = 0;
521
522 MapEvents(bh, br);
523
524 NB_next_pos = pos + (streampos)(bh.length<<2);
525 }
526
527 // Check if we did not find an event of interest above.
528 // If not, report that there are no more events in the file.
529 if(br.evio_events.empty() || !ifs.good()){
530 SetErrorMessage("No more events");
531 err_code = HDEVIO_EOF;
532 return false; // isgood=false
533 }
534
535 // Grab next event record
536 EVIOEventRecord &er = br.evio_events.front();
537
538 uint32_t event_len = er.event_len;
539 last_event_len = event_len;
540
541 // Check if user buffer is big enough to hold block
542 if( event_len > user_buff_len ){
543 ClearErrorMessage();
544 err_mess << "user buffer too small for event (" << user_buff_len << " < " << event_len << ")";
545 err_code = HDEVIO_USER_BUFFER_TOO_SMALL;
546 return false;
547 }
548
549 // Set file pointer to start of EVIO event (NOT block header!)
550 last_event_pos = er.pos;
551 ifs.seekg(last_event_pos, ios_base::beg);
552
553 // Read data directly into user buffer
554 ifs.read((char*)user_buff, event_len*sizeof(uint32_t));
555 if(!ifs.good()){
556 SetErrorMessage("No more events");
557 err_code = HDEVIO_EOF;
558 return false; // isgood=false
559 }
560
561 // Remove EVIO Event record, effectively advancing to
562 // next event for the next time we're called
563 br.evio_events.erase(NB_block_record.evio_events.begin());
564
565 // Swap entire bank if needed
566 swap_needed = br.swap_needed; // set flag in HDEVIO
567 bool isgood = true;
568 if(br.swap_needed && allow_swap){
569 uint32_t Nswapped = swap_bank(user_buff, user_buff, event_len);
570 isgood = (Nswapped == event_len);
571 }
572
573 // Double check that event length matches EVIO block header
574 // but only if we either don't need to swap or need to and
575 // were allowed to (otherwise, the test will almost certainly
576 // fail!)
577 if( (!br.swap_needed) || (br.swap_needed && allow_swap) ){
578 if( (user_buff[0]+1) != event_len ){
579 ClearErrorMessage();
580 err_mess << "WARNING: EVIO bank indicates a different size than block header (" << event_len << " != " << (user_buff[0]+1) << ")";
581 err_code = HDEVIO_EVENT_BIGGER_THAN_BLOCK;
582 Nerrors++;
583 Nbad_blocks++;
584 return false;
585 }
586 }
587
588 if(isgood) Nevents++;
589
590 return isgood;
591}
592
593//------------------------
594// rewind
595//------------------------
596void HDEVIO::rewind(void)
597{
598 /// This can be used whe reading from a file to
599 /// reset the file pointer and other position holders
600 /// to the begining of the file. This is done when
601 /// the "LOOP_FOREVER" option is used in the event source
602 /// to continuously re-read a file, essesntially making
603 /// it an infinite stream of events that can be used for
604 /// testing.
605
606 ifs.seekg(0, ios_base::beg);
607 ifs.clear();
608
609 sparse_block_iter = evio_blocks.begin();
610 sparse_event_idx = 0;
611
612 NB_block_record.evio_events.clear();
613 NB_next_pos = 0;
614
615 ClearErrorMessage();
616 err_code = HDEVIO_OK;
617}
618
619//------------------------
620// SetEventMask
621//------------------------
622uint32_t HDEVIO::SetEventMask(uint32_t mask)
623{
624 uint32_t prev_mask = event_type_mask;
625 event_type_mask = mask;
626
627 return prev_mask;
628}
629
630//------------------------
631// SetEventMask
632//------------------------
633uint32_t HDEVIO::SetEventMask(string types_str)
634{
635 uint32_t prev_mask = event_type_mask;
636
637 event_type_mask = 0;
638 if(types_str.find("BOR" ) != string::npos) event_type_mask |= (1<<kBT_BOR);
639 if(types_str.find("EPICS" ) != string::npos) event_type_mask |= (1<<kBT_EPICS);
640 if(types_str.find("PHYSICS") != string::npos) event_type_mask |= (1<<kBT_PHYSICS);
641
642 return prev_mask;
643}
644
645//------------------------
646// AddToEventMask
647//------------------------
648uint32_t HDEVIO::AddToEventMask(string type_str)
649{
650 return 0;
651}
652
653//------------------------
654// GetEVIOBlockRecords
655//------------------------
656vector<HDEVIO::EVIOBlockRecord>& HDEVIO::GetEVIOBlockRecords(void)
657{
658 if(!is_mapped) MapBlocks();
659
660 return evio_blocks;
661}
662
663//------------------------
664// MapBlocks
665//------------------------
666void HDEVIO::MapBlocks(bool print_ticker)
667{
668 if(!is_open){
669 err_mess.str("File is not open");
670 err_code = HDEVIO_FILE_NOT_OPEN;
671 return;
672 }
673
674 // Remember current file pos so we can restore it.
675 streampos start_pos = ifs.tellg();
676
677 if(print_ticker) cout << "Mapping EVIO file ..." << endl;
678
679 // Rewind to beginning of file and loop over all blocks
680 ifs.seekg(0, ios_base::beg);
681 BLOCKHEADER_t bh;
682 uint64_t Nblocks = 0;
683 while(ifs.good()){
684 ifs.read((char*)&bh, sizeof(bh));
685 if(!ifs.good()) break;
686
687 // Check if we need to byte swap and simultaneously
688 // verify header is good by checking magic word
689 bool swap_needed = false;
690 if(bh.magic==0x0001dac0){
691 swap_needed = true;
692 }else{
693 if(bh.magic!=0xc0da0100){
694 err_mess.str("Bad magic word");
695 err_code = HDEVIO_BAD_BLOCK_HEADER;
696 break;
697 }
698 }
699
700 if(swap_needed)swap_block((uint32_t*)&bh, sizeof(bh)>>2, (uint32_t*)&bh);
701
702 Nblocks++;
703 streampos block_len_bytes = (bh.length<<2)-sizeof(bh); // <<2 is for *4
704 streampos pos = ifs.tellg() - (streampos)sizeof(bh);
705
706 EVIOBlockRecord br;
707 br.pos = pos;
708 br.block_len = bh.length;
709 br.swap_needed = swap_needed;
710 br.first_event = 0;
711 br.last_event = 0;
712
713 // Categorize this block
714 uint32_t tag = bh.header>>16;
715 uint32_t M = bh.header&0xFF;
716
717 switch(tag){
718 case 0xFFD0: br.block_type = kBT_SYNC; break;
719 case 0xFFD1: br.block_type = kBT_PRESTART; break;
720 case 0xFFD2: br.block_type = kBT_GO; break;
721 case 0xFFD3: br.block_type = kBT_PAUSE; break;
722 case 0xFFD4: br.block_type = kBT_END; break;
723 case 0x0060: br.block_type = kBT_EPICS; break;
724 case 0x0070: br.block_type = kBT_BOR; break;
725 case 0xFF50:
726 case 0xFF51:
727 case 0xFF70:
728 br.block_type = kBT_PHYSICS;
729 br.first_event = bh.physics.first_event_lo;
730 br.first_event += ((uint64_t)bh.physics.first_event_hi)<<32;
731 br.last_event = br.first_event + (uint64_t)M - 1;
732 break;
733 default:
734 _DBG_cout<<"libraries/DAQ/HDEVIO.cc"<<":"<<734<<
" " << "Uknown tag: " << hex << tag << dec << endl;
735 }
736
737 // Scan through and map all events within this block
738 MapEvents(bh, br);
739
740 // Add block to list
741 evio_blocks.push_back(br);
742
743 // Update ticker
744 if(print_ticker){
745 if((Nblocks%500) == 0){
746 uint64_t total_MB = total_size_bytes>>20;
747 uint64_t read_MB = ifs.tellg()>>20;
748 if(Nblocks==0) cout << endl;
749 cout << Nblocks << " blocks scanned (" << read_MB << "/" << total_MB << " MB " << (100*read_MB/total_MB) << "%) \r";
750 cout.flush();
751 }
752 }
753
754 // Advance file pointer to start of next EVIO block header
755 ifs.seekg(block_len_bytes, ios_base::cur);
756 }
757
758 if(print_ticker) cout << endl;
759
760 // Setup iterators for sparse reading
761 sparse_block_iter = evio_blocks.begin();
762 sparse_event_idx = 0;
763
764 // Restore file pos and set flag that file has been mapped
765 ifs.clear();
766 ifs.seekg(start_pos, ios_base::beg);
767 is_mapped = true;
768}
769
770//---------------------------------
771// MapEvents
772//---------------------------------
773void HDEVIO::MapEvents(BLOCKHEADER_t &bh, EVIOBlockRecord &br)
774{
775 /// This is called if the EVIO block header indicates that
776 /// it contains more than one top-level event. The position
777 /// and event length of the first top-level event are passed
778 /// in as starting parameters.
779
780 // Record stream position upon entry so we can restore it at end
781 streampos start_pos = ifs.tellg();
782
783 // Calculate stream position of EVIO event
784 streampos pos = start_pos -(streampos)sizeof(BLOCKHEADER_t) + (streampos)(8<<2); // (8<<2) is 8 word EVIO block header times 4bytes/word
785 ifs.seekg(pos, ios_base::beg);
786
787 EVENTHEADER_t myeh;
788 for(uint32_t i=0; i<bh.eventcnt; i++){
789
790 // For the first iteration through this loop,
791 // we use the event header that has already been
792 // read in as part of the block header.
793 EVENTHEADER_t *eh = (EVENTHEADER_t*)&bh.event_len;
794 if(i!=0){
795 // Read in first few words of event
796 eh = &myeh;
797 ifs.read((char*)eh, sizeof(EVENTHEADER_t));
798 if(!ifs.good()) break;
799 if(br.swap_needed)swap_block((uint32_t*)eh, sizeof(EVENTHEADER_t)>>2, (uint32_t*)eh);
800 }else{
801 ifs.seekg(sizeof(EVENTHEADER_t), ios_base::cur);
802 }
803
804 if (eh->event_len < 2) {
805 // Before disabling this warning (or hiding it behind a VERBOSE flag)
806 // you should ask yourself the question, "Is this something that we
807 // should simply be ignoring, garbage bytes in the input evio file?"
808 std::cout << "HDEVIO::MapEvents warning - " << "Attempt to swap bank with len<2 (len="<<eh->event_len<<" header="<<hex<<eh->header<<dec<<" pos=" << pos << " tellg=" << ifs.tellg() << " i=" << i << ")" << std::endl;
809
810 // Reference run 20495: Seems ROL is putting BOR bank header, but
811 // the bank has no data in it. For this case we can go forward, but
812 // this is really a problem for production data. Detect this and warn
813 // user.
814 if(eh->event_len==1 && (eh->header&0xFFFF00FF)==0x00700001){
815 _DBG__cout<<"libraries/DAQ/HDEVIO.cc"<<":"<<815<<
endl;
816 _DBG_cout<<"libraries/DAQ/HDEVIO.cc"<<":"<<816<<
" " << "WARNING: This looks like an empty BOR event. BOR configuration" << endl;
817 _DBG_cout<<"libraries/DAQ/HDEVIO.cc"<<":"<<817<<
" " << " data will not be available and it is unlikely you will" << endl;
818 _DBG_cout<<"libraries/DAQ/HDEVIO.cc"<<":"<<818<<
" " << " be able to do anything beyond the digihit level. " << endl;
819 _DBG__cout<<"libraries/DAQ/HDEVIO.cc"<<":"<<819<<
endl;
820
821 if(IGNORE_EMPTY_BOR){
822 EVIOEventRecord er;
823 er.pos = pos;
824 er.event_len = eh->event_len + 1; // +1 to include length word
825 er.event_type = kBT_UNKNOWN;
826 er.first_event = 0;
827 er.last_event = 0;
828 br.evio_events.push_back(er);
829 }else{
830 _DBG_cout<<"libraries/DAQ/HDEVIO.cc"<<":"<<830<<
" " << " The program will (probably) stop now." << endl;
831 _DBG_cout<<"libraries/DAQ/HDEVIO.cc"<<":"<<831<<
" " << " To avoid stopping, re-run with EVIO:IGNORE_EMPTY_BOR=1 ." << endl; }
832 }
833
834 Nbad_events++;
835 Nerrors++;
836 // --i; // This caused an infinite loop when reading hd_rawdata_020058_000.evio DL
837 streampos delta = (streampos)((eh->event_len+1)<<2) -
838 (streampos)sizeof(EVENTHEADER_t);
839 ifs.seekg(delta, ios_base::cur);
840 pos += (streampos)((eh->event_len+1)<<2);
841 continue;
842 }
843
844 EVIOEventRecord er;
845 er.pos = pos;
846 er.event_len = eh->event_len + 1; // +1 to include length word
847 er.event_type = kBT_UNKNOWN;
848 er.first_event = 0;
849 er.last_event = 0;
850
851 uint32_t tag = eh->header>>16;
852 uint32_t M = eh->header&0xFF;
853
854 switch(tag){
855 case 0xFFD0: er.event_type = kBT_SYNC; break;
856 case 0xFFD1: er.event_type = kBT_PRESTART; break;
857 case 0xFFD2: er.event_type = kBT_GO; break;
858 case 0xFFD3: er.event_type = kBT_PAUSE; break;
859 case 0xFFD4: er.event_type = kBT_END; break;
860 case 0x0060: er.event_type = kBT_EPICS; break;
861 case 0x0070: er.event_type = kBT_BOR; break;
862 case 0xFF50:
863 case 0xFF51:
864 case 0xFF70:
865 er.event_type = kBT_PHYSICS;
866 er.first_event = eh->physics.first_event_lo;
867 er.first_event += ((uint64_t)eh->physics.first_event_hi)<<32;
868 er.last_event = er.first_event + (uint64_t)M - 1;
869 if(er.first_event < br.first_event) br.first_event = er.first_event;
870 if(er.last_event > br.last_event ) br.last_event = er.last_event;
871 break;
872 default:
873 _DBG_cout<<"libraries/DAQ/HDEVIO.cc"<<":"<<873<<
" " << "Uknown tag: " << hex << tag << dec << endl;
874 }
875
876 br.evio_events.push_back(er);
877
878 // Move file position to start of next event
879 streampos delta = (streampos)((eh->event_len+1)<<2) - (streampos)sizeof(EVENTHEADER_t);
880 ifs.seekg(delta, ios_base::cur);
881 pos += (streampos)((eh->event_len+1)<<2);
882 }
883
884 ifs.seekg(start_pos, ios_base::beg);
885}
886
887//---------------------------------
888// swap_bank
889//---------------------------------
890uint32_t HDEVIO::swap_bank(uint32_t *outbuff, uint32_t *inbuff, uint32_t len)
891{
892 /// Swap an EVIO bank. If the bank contains data, it is automatically
893 /// swapped according to it's type. If the bank is a container of other
894 /// containers, then this repeatedly calls the swapper methods for the
895 /// appropriate container type (bank, tagsegment, segment). This means
896 /// that this method will be recursive in the cases where it is a bank
897 /// of banks.
898
899 if(len < 2){
900 SetErrorMessage("Attempt to swap bank with len<2");
901 err_code = HDEVIO_BANK_TRUNCATED;
902 Nerrors++;
903 Nbad_events++;
904 return 0;
905 }
906
907 // Swap length and header words
908 swap_block(inbuff, 2, outbuff);
909 uint32_t bank_len = outbuff[0];
910 if((bank_len+1) > len){
911 ClearErrorMessage();
912 err_mess << "WARNING: Bank length word exceeds valid words in buffer (" << bank_len+1 << " > " << len << ")";
913 err_code = HDEVIO_BANK_TRUNCATED;
914 Nerrors++;
915 Nbad_events++;
916 return 0;
917 }
918
919 uint32_t type = (outbuff[1]>>8) & 0xFF;
920 uint32_t Nwords = bank_len - 1;
921 uint32_t Nswapped = 2;
922 switch(type){
923 case 0x0a: // 64 bit unsigned int
924 case 0x08: // 64 bit double
925 case 0x09: // 64 bit signed int
926 swap_block((uint64_t*)&inbuff[2], Nwords/2, (uint64_t*)&outbuff[2]);
927 Nswapped += Nwords;
928 break;
929 case 0x01: // 32 bit unsigned int
930 case 0x02: // 32 bit float
931 case 0x0b: // 32 bit signed int
932 swap_block(&inbuff[2], Nwords, &outbuff[2]);
933 Nswapped += Nwords;
934 break;
935 case 0x05: // 16 bit unsigned int
936 case 0x04: // 16 bit signed int
937 swap_block((uint16_t*)&inbuff[2], Nwords*2, (uint16_t*)&outbuff[2]);
938 Nswapped += Nwords;
939 break;
940 case 0x00: // 32 bit unknown (not swapped)
941 case 0x07: // 8 bit unsigned int
942 case 0x06: // 8 bit signed int
943 memcpy((uint8_t*)&outbuff[2], (uint8_t*)&inbuff[2], Nwords*sizeof(uint32_t));
944 Nswapped += Nwords;
945 break;
946 case 0x0c:
947 while(Nswapped < (Nwords+2)){
948 uint32_t N = swap_tagsegment(&outbuff[Nswapped], &inbuff[Nswapped], (Nwords+2)-Nswapped);
949 if(N == 0) return Nswapped;
950 Nswapped += N;
951 }
952 break;
953 case 0x0d:
954 case 0x20:
955 while(Nswapped < (Nwords+2)){
956 uint32_t N = swap_segment(&outbuff[Nswapped], &inbuff[Nswapped], (Nwords+2)-Nswapped);
957 if(N == 0) return Nswapped;
958 Nswapped += N;
959 }
960 break;
961 case 0x0e:
962 case 0x10:
963 while(Nswapped < (Nwords+2)){
964 uint32_t N = swap_bank(&outbuff[Nswapped], &inbuff[Nswapped], (Nwords+2)-Nswapped);
965 if(N == 0) return Nswapped;
966 Nswapped += N;
967 }
968 break;
969 default:
970 ClearErrorMessage();
971 err_mess << "WARNING: unknown bank type (0x" << hex << type << dec << ")";
972 err_code = HDEVIO_UNKNOWN_BANK_TYPE;
973 Nerrors++;
974 Nbad_events++;
975 return 0;
976 break;
977 }
978
979 return Nswapped;
980}
981
982//---------------------------------
983// swap_tagsegment
984//---------------------------------
985uint32_t HDEVIO::swap_tagsegment(uint32_t *outbuff, uint32_t *inbuff, uint32_t len)
986{
987 /// Swap an EVIO tagsegment.
988
989 if(len < 1){
990 SetErrorMessage("Attempt to swap segment with len<1");
991 err_code = HDEVIO_BANK_TRUNCATED;
992 Nerrors++;
993 Nbad_events++;
994 return 0;
995 }
996
997 // Swap header/length word
998 swap_block(inbuff, 1, outbuff);
999 uint32_t bank_len = outbuff[0] & 0xFFFF;
1000 if((bank_len) > len){
1001 ClearErrorMessage();
1002 err_mess << "Segment length word exceeds valid words in buffer (" << bank_len << " > " << len << ")";
1003 err_code = HDEVIO_BANK_TRUNCATED;
1004 Nerrors++;
1005 Nbad_events++;
1006 return 0;
1007 }
1008
1009 uint32_t type = (outbuff[0]>>16) & 0x0F;
1010 uint32_t Nwords = bank_len;
1011 uint32_t Nswapped = 1;
1012 switch(type){
1013 case 0x0a: // 64 bit unsigned int
1014 case 0x08: // 64 bit double
1015 case 0x09: // 64 bit signed int
1016 swap_block((uint64_t*)&inbuff[1], Nwords/2, (uint64_t*)&outbuff[1]);
1017 Nswapped += Nwords;
1018 break;
1019 case 0x01: // 32 bit unsigned int
1020 case 0x02: // 32 bit float
1021 case 0x0b: // 32 bit signed int
1022 swap_block(&inbuff[1], Nwords, &outbuff[1]);
1023 Nswapped += Nwords;
1024 break;
1025 case 0x05: // 16 bit unsigned int
1026 case 0x04: // 16 bit signed int
1027 swap_block((uint16_t*)&inbuff[1], Nwords*2, (uint16_t*)&outbuff[1]);
1028 Nswapped += Nwords;
1029 break;
1030 case 0x00: // 32 bit unknown (not swapped)
1031 case 0x07: // 8 bit unsigned int
1032 case 0x06: // 8 bit signed int
1033 memcpy((uint8_t*)&outbuff[1], (uint8_t*)&inbuff[1], Nwords*sizeof(uint32_t));
1034 Nswapped += Nwords;
1035 break;
1036 }
1037
1038 return Nswapped;
1039}
1040
1041//---------------------------------
1042// swap_segment
1043//---------------------------------
1044uint32_t HDEVIO::swap_segment(uint32_t *outbuff, uint32_t *inbuff, uint32_t len)
1045{
1046 /// Swap an EVIO segment.
1047
1048 if(len < 1){
1049 SetErrorMessage("Attempt to swap segment with len<1");
1050 err_code = HDEVIO_BANK_TRUNCATED;
1051 Nerrors++;
1052 Nbad_events++;
1053 return 0;
1054 }
1055
1056 // Swap header/length word
1057 swap_block(inbuff, 1, outbuff);
1058 uint32_t bank_len = outbuff[0] & 0xFFFF;
1059 if((bank_len) > len){
1060 ClearErrorMessage();
1061 err_mess << "Segment length word exceeds valid words in buffer (" << bank_len << " > " << len << ")";
1062 err_code = HDEVIO_BANK_TRUNCATED;
1063 Nerrors++;
1064 Nbad_events++;
1065 return 0;
1066 }
1067
1068 uint32_t type = (outbuff[0]>>16) & 0x3F;
1069 uint32_t Nwords = bank_len;
1070 uint32_t Nswapped = 1;
1071 switch(type){
1072 case 0x0a: // 64 bit unsigned int
1073 case 0x08: // 64 bit double
1074 case 0x09: // 64 bit signed int
1075 swap_block((uint64_t*)&inbuff[1], Nwords/2, (uint64_t*)&outbuff[1]);
1076 Nswapped += Nwords;
1077 break;
1078 case 0x01: // 32 bit unsigned int
1079 case 0x02: // 32 bit float
1080 case 0x0b: // 32 bit signed int
1081 swap_block(&inbuff[1], Nwords, &outbuff[1]);
1082 Nswapped += Nwords;
1083 break;
1084 case 0x05: // 16 bit unsigned int
1085 case 0x04: // 16 bit signed int
1086 swap_block((uint16_t*)&inbuff[1], Nwords*2, (uint16_t*)&outbuff[1]);
1087 Nswapped += Nwords;
1088 break;
1089 case 0x00: // 32 bit unknown (not swapped)
1090 case 0x07: // 8 bit unsigned int
1091 case 0x06: // 8 bit signed int
1092 memcpy((uint8_t*)&outbuff[1], (uint8_t*)&inbuff[1], Nwords*sizeof(uint32_t));
1093 Nswapped += Nwords;
1094 break;
1095 }
1096
1097 return Nswapped;
1098}
1099
1100//------------------------
1101// Print_fbuff
1102//------------------------
1103void HDEVIO::Print_fbuff(void)
1104{
1105 cout << endl;
1106 cout << " fbuff: " << hex << (uint64_t)fbuff << dec << endl;
1107 cout << " fnext: " << hex << (uint64_t)fnext << dec << endl;
1108 cout << " fbuff_end: " << hex << (uint64_t)fbuff_end << dec << endl;
1109 cout << " fbuff_size: " << fbuff_size << endl;
1110 cout << " fbuff_len: " << fbuff_len << endl;
1111 cout << " _gcount: " << _gcount << endl;
1112}
1113
1114//------------------------
1115// PrintEVIOBlockHeader
1116//------------------------
1117void HDEVIO::PrintEVIOBlockHeader(void)
1118{
1119
1120 cout << endl;
1121 cout << "EVIO Block Header:" << endl;
1122 cout << "------------------------" << endl;
1123 cout << " Block Length: " << HexStr(buff[0]) << " (" << buff[0] << " words = " << (buff[0]>>(10-2)) << " kB)" << endl;
1124 cout << " Block Number: " << HexStr(buff[1]) << endl;
1125 cout << "Header Length: " << HexStr(buff[2]) << " (should be 8)" << endl;
1126 cout << " Event Count: " << HexStr(buff[3]) << endl;
1127 cout << " Reserved 1: " << HexStr(buff[4]) << endl;
1128 cout << " Bit Info: " << HexStr(buff[5]>>8) << endl;
1129 cout << " Version: " << HexStr(buff[5]&0xFF) << endl;
1130 cout << " Reserved 3: " << HexStr(buff[6]) << endl;
1131 cout << " Magic word: " << HexStr(buff[7]) << (swap_needed ? " (after swapping)":"") << endl;
1132 cout << "Byte swapping is" << (swap_needed ? " ":" not ") << "needed" << endl;
1133}
1134
1135//------------------------
1136// PrintStats
1137//------------------------
1138void HDEVIO::PrintStats(void)
1139{
1140 cout << endl;
1141 cout << "EVIO Statistics for " << filename << " :" << endl;
1142 cout << "------------------------" << endl;
1143 cout << " Nblocks: " << Nblocks << endl;
1144 cout << " Nevents: " << Nevents << endl;
1145 cout << " Nerrors: " << Nerrors << endl;
1146 cout << "Nbad_blocks: " << Nbad_blocks << endl;
1147 cout << "Nbad_events: " << Nbad_events << endl;
1148 cout << endl;
1149}
1150
1151//------------------------
1152// PrintFileSummary
1153//------------------------
1154void HDEVIO::PrintFileSummary(void)
1155{
1156 if(!is_mapped) MapBlocks();
1157
1158 uint32_t Nsync = 0;
1159 uint32_t Nprestart = 0;
1160 uint32_t Ngo = 0;
1161 uint32_t Npause = 0;
1162 uint32_t Nend = 0;
1163 uint32_t Nepics = 0;
1164 uint32_t Nbor = 0;
1165 uint32_t Nphysics = 0;
1166
1167 uint64_t first_event = 0;
1168 uint64_t last_event = 0;
1169
1170 uint32_t map_size = evio_blocks.size()*sizeof(EVIOBlockRecord);
1171
1172 set<uint32_t> block_levels;
1173 set<uint32_t> events_in_block;
1174
1175 // Loop over all EVIO block records
1176 for(uint32_t i=0; i<evio_blocks.size(); i++){
1177 EVIOBlockRecord &br = evio_blocks[i];
1178 events_in_block.insert(br.evio_events.size());
1179 map_size += br.evio_events.size()*sizeof(EVIOEventRecord);
1180
1181 for(uint32_t j=0; j<br.evio_events.size(); j++){
1182 EVIOEventRecord &er = br.evio_events[j];
1183
1184 uint32_t block_level;
1185 switch(er.event_type){
1186 case kBT_SYNC: Nsync++; break;
1187 case kBT_PRESTART: Nprestart++; break;
1188 case kBT_GO: Ngo++; break;
1189 case kBT_PAUSE: Npause++; break;
1190 case kBT_END: Nend++; break;
1191 case kBT_EPICS: Nepics++; break;
1192 case kBT_BOR: Nbor++; break;
1193 case kBT_PHYSICS:
1194 block_level = (uint32_t)((er.last_event - er.first_event) + 1);
1195 block_levels.insert(block_level);
1196 Nphysics += block_level;
1197 if(er.first_event<first_event || first_event==0) first_event = er.first_event;
1198 if(er.last_event>last_event) last_event = er.last_event;
1199 break;
1200 default:
1201 break;
1202 }
1203
1204 //_DBG_ << "Block " << i << " event " << j << " " << er.last_event <<" - " << er.first_event << " = " << block_level << endl;
1205 }
1206 }
1207
1208 // form succint string of block levels
1209 stringstream ss;
1210 set<uint32_t>::iterator it = block_levels.begin();
1211 for(; it!=block_levels.end(); it++) ss << *it << ",";
1212 string sblock_levels = ss.str();
1213 if(!sblock_levels.empty()) sblock_levels.erase(sblock_levels.length()-1);
1214
1215 // form succint string of events per block
1216 ss.str("");
1217 it = events_in_block.begin();
1218 for(; it!=events_in_block.end(); it++){
1219 uint32_t val = *it;
1220 ss << val;
1221 if(++it==events_in_block.end()) break;
1222 if( *it == ++val ){
1223 ss << "-";
1224 for(it++; it!=events_in_block.end(); it++){
1225 if( *it != ++val ){
1226 ss << (val-1) << ",";
1227 it--;
1228 break;
1229 }
1230 }
1231 }else{
1232 ss << ",";
1233 }
1234 if( it==events_in_block.end() ) break;
1235 }
1236 string sevents_in_block = ss.str();
1237
1238 // Print results
1239 cout << endl;
1240 cout << "EVIO file size: " << (total_size_bytes>>20) << " MB" <<endl;
1241 cout << "EVIO block map size: " << (map_size>>10) << " kB" <<endl;
1242 cout << "first event: " << first_event << endl;
1243 cout << "last event: " << last_event << endl;
1244
1245 cout << endl;
1246 cout << " Nblocks = " << evio_blocks.size() << endl;
1247 cout << " block levels = " << sblock_levels << endl;
1248 cout << "events per block = " << sevents_in_block << endl;
1249 cout << " Nsync = " << Nsync << endl;
1250 cout << " Nprestart = " << Nprestart << endl;
1251 cout << " Ngo = " << Ngo << endl;
1252 cout << " Npause = " << Npause << endl;
1253 cout << " Nend = " << Nend << endl;
1254 cout << " Nepics = " << Nepics << endl;
1255 cout << " Nbor = " << Nbor << endl;
1256 cout << " Nphysics = " << Nphysics << endl;
1257 cout << endl;
1258}
1259
1260