Line data Source code
1 : /** \file readColumns.hpp
2 : * \author Jared R. Males
3 : * \brief A utility to read in columns from a text file.
4 : * \ingroup asciiutils
5 : */
6 :
7 : //***********************************************************************//
8 : // Copyright 2015, 2016, 2017 Jared R. Males (jaredmales@gmail.com)
9 : //
10 : // This file is part of mxlib.
11 : //
12 : // mxlib is free software: you can redistribute it and/or modify
13 : // it under the terms of the GNU General Public License as published by
14 : // the Free Software Foundation, either version 3 of the License, or
15 : // (at your option) any later version.
16 : //
17 : // mxlib is distributed in the hope that it will be useful,
18 : // but WITHOUT ANY WARRANTY; without even the implied warranty of
19 : // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 : // GNU General Public License for more details.
21 : //
22 : // You should have received a copy of the GNU General Public License
23 : // along with mxlib. If not, see <http://www.gnu.org/licenses/>.
24 : //***********************************************************************//
25 :
26 : #ifndef __readColumns_hpp__
27 : #define __readColumns_hpp__
28 :
29 : #include <cctype>
30 : #include <cstring>
31 : #include <fstream>
32 : #include <format>
33 : #include <string>
34 :
35 : #include "../mxlib.hpp"
36 :
37 : #include "stringUtils.hpp"
38 :
39 : #define MX_READCOL_MISSINGVALSTR "-99"
40 :
41 : namespace mx
42 : {
43 : namespace ioutils
44 : {
45 :
46 : #ifndef ioutils_readColumns_detail_hpp
47 : #define ioutils_readColumns_detail_hpp
48 : namespace readColumnsDetail
49 : {
50 :
51 : /** \cond */
52 : /// File-stream stages exposed to deterministic exception and error tests.
53 : enum class operation
54 : {
55 : afterOpen,
56 : afterReadLine,
57 : afterClose
58 : };
59 :
60 : /// Signature of the resettable file-stream operation hook.
61 : using operationHookT = void ( * )( operation, std::ifstream & );
62 :
63 : /// Production no-op for the file-stream operation hook.
64 64 : inline void noOperationFailure( operation, std::ifstream & )
65 : {
66 64 : }
67 :
68 : /// Access the process-wide file-stream operation hook.
69 102 : inline operationHookT &operationHook()
70 : {
71 : static operationHookT hook = noOperationFailure;
72 102 : return hook;
73 : }
74 : /** \endcond */
75 :
76 : } // namespace readColumnsDetail
77 : #endif // ioutils_readColumns_detail_hpp
78 :
79 : #ifdef MXLIBTEST_NAMESPACE
80 : namespace MXLIBTEST_NAMESPACE
81 : {
82 : #endif
83 :
84 : struct readColSpaceDelim
85 : {
86 : static constexpr char delim = ' ';
87 : static constexpr char strDelim = '"';
88 : static constexpr char eol = '\n';
89 : static constexpr char comment = '#';
90 : static constexpr const char *missingValStr = MX_READCOL_MISSINGVALSTR;
91 : };
92 :
93 : struct readColCommaDelim
94 : {
95 : static constexpr char delim = ',';
96 : static constexpr char strdelim = '"';
97 : static constexpr char eol = '\n';
98 : static constexpr char comment = '#';
99 : static constexpr const char *missingValStr = MX_READCOL_MISSINGVALSTR;
100 : };
101 :
102 : template <class delimT, class verboseT>
103 37 : error_t readcol( [[maybe_unused]] const char *sin, [[maybe_unused]] int sz, [[maybe_unused]] int &colno )
104 : {
105 37 : return error_t::noerror;
106 : }
107 :
108 : template <class delimT, class verboseT, typename arrT, typename... arrTs>
109 82 : error_t readcol( const char *sin, int sz, int &colno, arrT &array, arrTs &...arrays )
110 : {
111 : try
112 : {
113 82 : std::string str;
114 :
115 82 : int i = 0;
116 82 : int l = strlen( sin );
117 :
118 82 : if( l < 1 )
119 : {
120 1 : return error_t::noerror;
121 : }
122 :
123 : // Eat white space
124 87 : while( i < l && std::isspace( static_cast<unsigned char>( sin[i] ) ) && sin[i] != delimT::eol )
125 : {
126 6 : ++i;
127 : }
128 81 : sin = sin + i;
129 81 : sz = sz - i;
130 :
131 : // If there's nothing here, we still need to populate the vector
132 81 : if( sz == 0 )
133 : {
134 : mx::error_t errc;
135 2 : array.push_back( stoT<typename arrT::value_type>( "", &errc ) );
136 :
137 2 : if( errc != mx::error_t::noerror )
138 : {
139 1 : return internal::mxlib_error_report<verboseT>( errc, std::format( "processing column {}", colno ) );
140 : }
141 :
142 1 : return mx::error_t::noerror;
143 : }
144 :
145 158 : std::stringstream sinstr( sin );
146 :
147 79 : std::getline( sinstr, str, delimT::delim );
148 :
149 : // Last entry in line might contain eol
150 79 : if( !str.empty() && str[str.size() - 1] == delimT::eol )
151 : {
152 1 : str.erase( str.size() - 1 );
153 : }
154 :
155 : mx::error_t errc;
156 79 : if( str.size() == 0 )
157 : {
158 2 : array.push_back( stoT<typename arrT::value_type>( MX_READCOL_MISSINGVALSTR, &errc ) );
159 : }
160 : else
161 : {
162 86 : array.push_back( stoT<typename arrT::value_type>( str, &errc ) );
163 : }
164 :
165 71 : if( errc != mx::error_t::noerror )
166 : {
167 1 : return internal::mxlib_error_report<verboseT>( errc, std::format( "processing column {}", colno ) );
168 : }
169 :
170 70 : sin += ( str.size() + 1 ) * sizeof( char );
171 70 : sz -= ( str.size() + 1 ) * sizeof( char );
172 91 : }
173 8 : catch( const std::invalid_argument &e )
174 : {
175 : // We always catch this one
176 1 : return internal::mxlib_error_report<verboseT>( error_t::std_invalid_argument,
177 2 : std::format( "processing column {}: {}", colno, e.what() ) );
178 : }
179 1 : catch( const std::out_of_range &e )
180 : {
181 : // We always catch this one
182 1 : return internal::mxlib_error_report<verboseT>( error_t::std_out_of_range,
183 2 : std::format( "processing column {}: {}", colno, e.what() ) );
184 : }
185 3 : catch( const std::bad_alloc &e )
186 : {
187 :
188 : // clang-format off
189 : #if defined( MXLIB_CATCH_ALL_EXCEPTIONS )
190 1 : return internal::mxlib_error_report<verboseT>( error_t::std_bad_alloc,
191 2 : std::format( "processing column {}: {}", colno, e.what() ) );
192 : #else
193 2 : std::throw_with_nested(mx::exception<verboseT>(error_t::std_bad_alloc,
194 3 : std::format( "processing column {}: {}", colno, e.what() ) ));
195 : #endif
196 : // clang-format on
197 : }
198 3 : catch( const std::exception &e )
199 : {
200 : // clang-format off
201 : #if defined( MXLIB_CATCH_ALL_EXCEPTIONS ) || defined( MXLIB_CATCH_NONALLOC_EXCEPTIONS )
202 1 : return internal::mxlib_error_report<verboseT>( error_t::std_exception,
203 2 : std::format( "processing column {}: {}", colno, e.what() ) );
204 :
205 : #else
206 2 : std::throw_with_nested(mx::exception<verboseT>(error_t::std_exception,
207 3 : std::format( "processing column {}: {}", colno, e.what() ) ));
208 :
209 : #endif
210 : // clang-format on
211 : }
212 3 : catch( ... )
213 : {
214 : // clang-format off
215 : #if defined( MXLIB_CATCH_ALL_EXCEPTIONS ) || defined( MXLIB_CATCH_NONALLOC_EXCEPTIONS )
216 1 : return internal::mxlib_error_report<verboseT>( error_t::exception,
217 2 : std::format( "processing column {}", colno) );
218 :
219 : #else
220 2 : std::throw_with_nested(mx::exception<verboseT>(error_t::exception,
221 3 : std::format( "processing column {}", colno) ));
222 :
223 : #endif
224 : }
225 :
226 70 : ++colno;
227 70 : return readcol<delimT, verboseT>( sin, sz, colno, arrays... );
228 : }
229 :
230 : /// Read in columns from a text file
231 : /** This function opens a file containing data formatted in columns and reads in the data row by row.
232 : * The data are stored in std::vectors, which should not be pre-allocated (though they could be reserve()-ed).
233 : *
234 : * Example:
235 : * \code
236 : * std::vector<int> i1;
237 : * std::vector<float> f1;
238 : * std::vector<double> d1;
239 : *
240 : * readColumns("data_file.txt", i1, f1, d1);
241 : * \endcode
242 : *
243 : * Note that the types of the vectors do not need to be specified as template arguments.
244 : *
245 : * The format of the file can be specified with template arguments like
246 : * \code
247 : * readColumns<',', ';', '\r'>("data_file.csv", i1, f1, d1);
248 : * \endcode
249 : * which sets the delimmiter to comma, the comment character to ;, and the end-of-line to \\r.
250 : *
251 : * Columns can be skipped using mx::ioutils::skipCol.
252 : *
253 : * \tparam delimT specifies the delimiters. By default this is `mx::ioutils::readColSpaceDelim`.
254 : * \tparam verbose specifies the error reporting verbosity. See \ref mx::verbose
255 : *
256 : *
257 : * \ingroup asciiutils
258 : */
259 : template <class delimT = readColSpaceDelim, class verboseT = verbose::d, typename... arrTs>
260 21 : error_t readColumns( const std::string &fname, ///< [in] is the file name to read from
261 : arrTs &...arrays /**< [out] a variadic list of std::vectors. Any number with mixed
262 : value_type can be specified. Neither allocated nor cleared,
263 : so repeated calls will append data.*/
264 : )
265 : {
266 : // open file
267 21 : errno = 0;
268 21 : std::ifstream fin;
269 21 : fin.open( fname );
270 21 : readColumnsDetail::operationHook()( readColumnsDetail::operation::afterOpen, fin );
271 :
272 21 : if( !fin.good() )
273 : {
274 : error_t errc;
275 3 : if( errno != 0 )
276 : {
277 2 : errc = errno2error_t( errno );
278 : }
279 : else
280 : {
281 1 : errc = error_t::fileoerr;
282 : }
283 :
284 3 : return internal::mxlib_error_report<verboseT>( errc, "Opening " + fname + " for reading" );
285 : }
286 :
287 18 : std::string line;
288 :
289 18 : int64_t lineno = -1;
290 :
291 69 : while( fin.good() )
292 : {
293 60 : ++lineno;
294 : try
295 : {
296 60 : std::getline( fin, line, delimT::eol );
297 60 : readColumnsDetail::operationHook()( readColumnsDetail::operation::afterReadLine, fin );
298 : }
299 7 : catch( const std::bad_alloc &e )
300 : {
301 :
302 : // clang-format off
303 : #if defined( MXLIB_CATCH_ALL_EXCEPTIONS )
304 1 : return internal::mxlib_error_report<verboseT>( error_t::std_bad_alloc,
305 1 : std::format( "Reading from {} at line {}: {}.",
306 3 : fname, lineno, e.what() ) );
307 : ;
308 : #else
309 2 : std::throw_with_nested( mx::exception<verboseT>( error_t::std_bad_alloc,
310 3 : std::format( "Reading from {} at line {}: {}.",
311 2 : fname, lineno, e.what() ) ) );
312 : #endif
313 : // clang-format on
314 : }
315 3 : catch( const std::exception &e )
316 : {
317 :
318 : // clang-format off
319 : #if defined( MXLIB_CATCH_ALL_EXCEPTIONS ) || defined(MXLIB_CATCH_NONALLOC_EXCEPTIONS)
320 :
321 1 : return internal::mxlib_error_report<verboseT>( error_t::std_exception,
322 1 : std::format( "Reading from {} at line {}: {}.",
323 : fname,
324 : lineno,
325 3 : e.what() ) );
326 : #else
327 2 : std::throw_with_nested(mx::exception<verboseT>( error_t::std_exception,
328 3 : std::format( "Reading from {} at line {}: {}.",
329 2 : fname, lineno, e.what() ) ) );
330 : #endif
331 : // clang-format on
332 : }
333 3 : catch( ... )
334 : {
335 :
336 : // clang-format off
337 : #if defined( MXLIB_CATCH_ALL_EXCEPTIONS ) || defined(MXLIB_CATCH_NONALLOC_EXCEPTIONS)
338 :
339 1 : return internal::mxlib_error_report<verboseT>( error_t::exception,
340 2 : std::format( "Reading from {} at line {}",
341 : fname,
342 1 : lineno ) );
343 : #else
344 2 : std::throw_with_nested(mx::exception<verboseT>( error_t::exception,
345 3 : std::format( "Reading from {} at line {}",
346 : fname, lineno) ) );
347 : #endif
348 : // clang-format on
349 : }
350 :
351 54 : if( fin.bad() )
352 : {
353 2 : break;
354 : }
355 :
356 52 : if( line.size() == 0 )
357 : {
358 16 : continue;
359 : }
360 :
361 : // Find start of comment and end line at that point.
362 45 : size_t i = 0;
363 45 : bool nonspace = false; // record if we find a non-space character before the comment
364 437 : while( i < line.size() && line[i] != delimT::comment )
365 : {
366 392 : if( !nonspace && !std::isspace( static_cast<unsigned char>( line[i] ) ) )
367 : {
368 36 : nonspace = true;
369 : }
370 392 : ++i;
371 : }
372 :
373 : // Check if line is all comment
374 45 : if( i == 0 || !nonspace )
375 : {
376 9 : continue;
377 : }
378 :
379 36 : if( i < line.size() ) // i is > 0 if we're here
380 : {
381 20 : line.erase( line.begin() + i, line.end() ); // does not throw
382 : }
383 :
384 36 : int colno = 0;
385 36 : error_t errc = readcol<delimT, verboseT>( line.c_str(), line.size(), colno, arrays... );
386 :
387 36 : if( errc != error_t::noerror )
388 : {
389 1 : return internal::mxlib_error_report<verboseT>(
390 : errc,
391 2 : std::format( "Reading from {} at line {} column {}", fname, lineno + 1, colno + 1 ) );
392 : }
393 : }
394 :
395 : // getline will have set fail if there was no new line on the last line.
396 11 : if( fin.bad() )
397 : {
398 : error_t errc;
399 2 : if( errno != 0 )
400 : {
401 1 : errc = errno2error_t( errno );
402 : }
403 : else
404 : {
405 1 : errc = error_t::filererr;
406 : }
407 :
408 2 : return internal::mxlib_error_report<verboseT>( errc, "Reading from " + fname );
409 : }
410 :
411 9 : fin.clear(); // Clear the fail bit which may have been set by getline
412 9 : errno = 0;
413 9 : fin.close();
414 9 : readColumnsDetail::operationHook()( readColumnsDetail::operation::afterClose, fin );
415 :
416 9 : if( fin.fail() )
417 : {
418 : error_t errc;
419 2 : if( errno != 0 )
420 : {
421 1 : errc = errno2error_t( errno );
422 : }
423 : else
424 : {
425 1 : errc = error_t::filecerr;
426 : }
427 :
428 2 : return internal::mxlib_error_report<verboseT>( errc, "Closing" + fname );
429 : }
430 :
431 7 : return error_t::noerror;
432 24 : }
433 :
434 : /// A dummy class to allow mx::readColumns to skip a column(s) in a file without requiring memory allocation.
435 : /** The alternative is to use dummy vectors, which result in excess memory allocations and deallocations.
436 : * Usage:
437 : \code
438 : std::vector<T> col1, col5;
439 : skipCol sk;
440 : readColumns("filename.txt", col1, sk, sk, sk, col5); //This results in only columns 1 and 5 being stored.
441 : \endcode
442 : *
443 : * \ingroup asciiutils
444 : */
445 : struct skipCol
446 : {
447 : typedef std::string value_type; ///< value_type is defined as std::string so that no conversions take place.
448 :
449 : template <typename T>
450 1 : void push_back( const T &arg )
451 : {
452 1 : return;
453 : }
454 : };
455 :
456 : #ifdef MXLIBTEST_NAMESPACE
457 : } // namespace MXLIBTEST_NAMESPACE
458 : #endif
459 :
460 : } // namespace ioutils
461 : } // namespace mx
462 :
463 : #endif //__readColumns_hpp__
|
