1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.commons.csv;
19
20 import static org.apache.commons.csv.Constants.CR;
21 import static org.apache.commons.csv.Constants.CRLF;
22 import static org.apache.commons.csv.Constants.LF;
23 import static org.junit.jupiter.api.Assertions.assertArrayEquals;
24 import static org.junit.jupiter.api.Assertions.assertEquals;
25 import static org.junit.jupiter.api.Assertions.assertFalse;
26 import static org.junit.jupiter.api.Assertions.assertNotNull;
27 import static org.junit.jupiter.api.Assertions.assertNull;
28 import static org.junit.jupiter.api.Assertions.assertThrows;
29 import static org.junit.jupiter.api.Assertions.assertTrue;
30
31 import java.io.File;
32 import java.io.IOException;
33 import java.io.InputStreamReader;
34 import java.io.PipedReader;
35 import java.io.PipedWriter;
36 import java.io.Reader;
37 import java.io.StringReader;
38 import java.io.StringWriter;
39 import java.io.UncheckedIOException;
40 import java.net.URL;
41 import java.nio.charset.Charset;
42 import java.nio.charset.StandardCharsets;
43 import java.nio.file.Files;
44 import java.nio.file.Path;
45 import java.nio.file.Paths;
46 import java.util.ArrayList;
47 import java.util.Arrays;
48 import java.util.Iterator;
49 import java.util.List;
50 import java.util.Map;
51 import java.util.NoSuchElementException;
52 import java.util.stream.Collectors;
53 import java.util.stream.Stream;
54
55 import org.apache.commons.io.input.BOMInputStream;
56 import org.apache.commons.io.input.BrokenInputStream;
57 import org.apache.commons.lang3.stream.Streams.FailableStream;
58 import org.junit.jupiter.api.Assertions;
59 import org.junit.jupiter.api.Disabled;
60 import org.junit.jupiter.api.Test;
61 import org.junit.jupiter.params.ParameterizedTest;
62 import org.junit.jupiter.params.provider.EnumSource;
63 import org.junit.jupiter.params.provider.ValueSource;
64
65
66
67
68
69
70
71
72 public class CSVParserTest {
73
74 private static final Charset UTF_8 = StandardCharsets.UTF_8;
75
76 private static final String UTF_8_NAME = UTF_8.name();
77
78 private static final String CSV_INPUT = "a,b,c,d\n" + " a , b , 1 2 \n" + "\"foo baar\", b,\n"
79
80 + " \"foo\n,,\n\"\",,\n\"\"\",d,e\n";
81
82 private static final String CSV_INPUT_1 = "a,b,c,d";
83
84 private static final String CSV_INPUT_2 = "a,b,1 2";
85
86 private static final String[][] RESULT = {{"a", "b", "c", "d"}, {"a", "b", "1 2"}, {"foo baar", "b", ""}, {"foo\n,,\n\",,\n\"", "d", "e"}};
87
88
89 static private final String CSV_INPUT_NO_COMMENT = "A,B"+CRLF+"1,2"+CRLF;
90
91
92 static private final String CSV_INPUT_HEADER_COMMENT = "# header comment" + CRLF + "A,B" + CRLF + "1,2" + CRLF;
93
94
95 static private final String CSV_INPUT_HEADER_TRAILER_COMMENT = "# header comment" + CRLF + "A,B" + CRLF + "1,2" + CRLF + "# comment";
96
97
98 static private final String CSV_INPUT_MULTILINE_HEADER_TRAILER_COMMENT = "# multi-line" + CRLF + "# header comment" + CRLF + "A,B" + CRLF + "1,2" + CRLF + "# multi-line" + CRLF + "# comment";
99
100
101 static private final CSVFormat FORMAT_AUTO_HEADER = CSVFormat.Builder.create(CSVFormat.DEFAULT).setCommentMarker('#').setHeader().build();
102
103
104 static private final CSVFormat FORMAT_EXPLICIT_HEADER = CSVFormat.Builder.create(CSVFormat.DEFAULT)
105 .setSkipHeaderRecord(true)
106 .setCommentMarker('#')
107 .setHeader("A", "B")
108 .build();
109
110
111 CSVFormat FORMAT_EXPLICIT_HEADER_NOSKIP = CSVFormat.Builder.create(CSVFormat.DEFAULT)
112 .setCommentMarker('#')
113 .setHeader("A", "B")
114 .build();
115
116 @SuppressWarnings("resource")
117 private BOMInputStream createBOMInputStream(final String resource) throws IOException {
118 return new BOMInputStream(ClassLoader.getSystemClassLoader().getResource(resource).openStream());
119 }
120
121 CSVRecord parse(final CSVParser parser, final int failParseRecordNo) throws IOException {
122 if (parser.getRecordNumber() + 1 == failParseRecordNo) {
123 assertThrows(IOException.class, () -> parser.nextRecord());
124 return null;
125 }
126 return parser.nextRecord();
127 }
128
129 private void parseFully(final CSVParser parser) {
130 parser.forEach(Assertions::assertNotNull);
131 }
132
133 @Test
134 public void testBackslashEscaping() throws IOException {
135
136
137
138
139
140 final String code = "one,two,three\n"
141 + "'',''\n"
142 + "/',/'\n"
143 + "'/'','/''\n"
144 + "'''',''''\n"
145 + "/,,/,\n"
146 + "//,//\n" // 6) escape escaped
147 + "'//','//'\n" // 7) escape escaped in encapsulation
148 + " 8 , \"quoted \"\" /\" // string\" \n" // don't eat spaces
149 + "9, /\n \n"
150 + "";
151 final String[][] res = {{"one", "two", "three"},
152 {"", ""},
153 {"'", "'"},
154 {"'", "'"},
155 {"'", "'"},
156 {",", ","},
157 {"/", "/"},
158 {"/", "/"},
159 {" 8 ", " \"quoted \"\" /\" / string\" "}, {"9", " \n "},};
160
161 final CSVFormat format = CSVFormat.newFormat(',').withQuote('\'').withRecordSeparator(CRLF).withEscape('/').withIgnoreEmptyLines();
162
163 try (final CSVParser parser = CSVParser.parse(code, format)) {
164 final List<CSVRecord> records = parser.getRecords();
165 assertFalse(records.isEmpty());
166
167 Utils.compare("Records do not match expected result", res, records);
168 }
169 }
170
171 @Test
172 public void testBackslashEscaping2() throws IOException {
173
174
175
176
177
178 final String code = "" + " , , \n"
179 + " \t , , \n"
180 + " // , /, , /,\n" // 3)
181 + "";
182 final String[][] res = {{" ", " ", " "},
183 {" \t ", " ", " "},
184 {" / ", " , ", " ,"},
185 };
186
187 final CSVFormat format = CSVFormat.newFormat(',').withRecordSeparator(CRLF).withEscape('/').withIgnoreEmptyLines();
188
189 try (final CSVParser parser = CSVParser.parse(code, format)) {
190 final List<CSVRecord> records = parser.getRecords();
191 assertFalse(records.isEmpty());
192
193 Utils.compare("", res, records);
194 }
195 }
196
197 @Test
198 @Disabled
199 public void testBackslashEscapingOld() throws IOException {
200 final String code = "one,two,three\n" + "on\\\"e,two\n" + "on\"e,two\n" + "one,\"tw\\\"o\"\n" + "one,\"t\\,wo\"\n" + "one,two,\"th,ree\"\n"
201 + "\"a\\\\\"\n" + "a\\,b\n" + "\"a\\\\,b\"";
202 final String[][] res = {{"one", "two", "three"}, {"on\\\"e", "two"}, {"on\"e", "two"}, {"one", "tw\"o"}, {"one", "t\\,wo"},
203
204 {"one", "two", "th,ree"}, {"a\\\\"},
205 {"a\\", "b"},
206 {"a\\\\,b"}
207 };
208 try (final CSVParser parser = CSVParser.parse(code, CSVFormat.DEFAULT)) {
209 final List<CSVRecord> records = parser.getRecords();
210 assertEquals(res.length, records.size());
211 assertFalse(records.isEmpty());
212 for (int i = 0; i < res.length; i++) {
213 assertArrayEquals(res[i], records.get(i).values());
214 }
215 }
216 }
217
218 @Test
219 @Disabled("CSV-107")
220 public void testBOM() throws IOException {
221 final URL url = ClassLoader.getSystemClassLoader().getResource("org/apache/commons/csv/CSVFileParser/bom.csv");
222 try (final CSVParser parser = CSVParser.parse(url, StandardCharsets.UTF_8, CSVFormat.EXCEL.withHeader())) {
223 parser.forEach(record -> assertNotNull(record.get("Date")));
224 }
225 }
226
227 @Test
228 public void testBOMInputStreamParserWithInputStream() throws IOException {
229 try (final BOMInputStream inputStream = createBOMInputStream("org/apache/commons/csv/CSVFileParser/bom.csv");
230 final CSVParser parser = CSVParser.parse(inputStream, UTF_8, CSVFormat.EXCEL.withHeader())) {
231 parser.forEach(record -> assertNotNull(record.get("Date")));
232 }
233 }
234
235 @Test
236 public void testBOMInputStreamParserWithReader() throws IOException {
237 try (final Reader reader = new InputStreamReader(createBOMInputStream("org/apache/commons/csv/CSVFileParser/bom.csv"), UTF_8_NAME);
238 final CSVParser parser = new CSVParser(reader, CSVFormat.EXCEL.withHeader())) {
239 parser.forEach(record -> assertNotNull(record.get("Date")));
240 }
241 }
242
243 @Test
244 public void testBOMInputStreamParseWithReader() throws IOException {
245 try (final Reader reader = new InputStreamReader(createBOMInputStream("org/apache/commons/csv/CSVFileParser/bom.csv"), UTF_8_NAME);
246 final CSVParser parser = CSVParser.parse(reader, CSVFormat.EXCEL.withHeader())) {
247 parser.forEach(record -> assertNotNull(record.get("Date")));
248 }
249 }
250
251 @Test
252 public void testCarriageReturnEndings() throws IOException {
253 final String code = "foo\rbaar,\rhello,world\r,kanu";
254 try (final CSVParser parser = CSVParser.parse(code, CSVFormat.DEFAULT)) {
255 final List<CSVRecord> records = parser.getRecords();
256 assertEquals(4, records.size());
257 }
258 }
259
260 @Test
261 public void testCarriageReturnLineFeedEndings() throws IOException {
262 final String code = "foo\r\nbaar,\r\nhello,world\r\n,kanu";
263 try (final CSVParser parser = CSVParser.parse(code, CSVFormat.DEFAULT)) {
264 final List<CSVRecord> records = parser.getRecords();
265 assertEquals(4, records.size());
266 }
267 }
268
269 @Test
270 public void testClose() throws Exception {
271 final Reader in = new StringReader("# comment\na,b,c\n1,2,3\nx,y,z");
272 final Iterator<CSVRecord> records;
273 try (final CSVParser parser = CSVFormat.DEFAULT.withCommentMarker('#').withHeader().parse(in)) {
274 records = parser.iterator();
275 assertTrue(records.hasNext());
276 }
277 assertFalse(records.hasNext());
278 assertThrows(NoSuchElementException.class, records::next);
279 }
280
281 @Test
282 public void testCSV141CSVFormat_DEFAULT() throws Exception {
283 testCSV141Failure(CSVFormat.DEFAULT, 3);
284 }
285
286 @Test
287 public void testCSV141CSVFormat_INFORMIX_UNLOAD() throws Exception {
288 testCSV141Failure(CSVFormat.INFORMIX_UNLOAD, 1);
289 }
290
291 @Test
292 public void testCSV141CSVFormat_INFORMIX_UNLOAD_CSV() throws Exception {
293 testCSV141Failure(CSVFormat.INFORMIX_UNLOAD_CSV, 3);
294 }
295
296 @Test
297 public void testCSV141CSVFormat_ORACLE() throws Exception {
298 testCSV141Failure(CSVFormat.ORACLE, 2);
299 }
300
301 @Test
302 public void testCSV141CSVFormat_POSTGRESQL_CSV() throws Exception {
303 testCSV141Failure(CSVFormat.POSTGRESQL_CSV, 3);
304 }
305
306 @Test
307 public void testCSV141Excel() throws Exception {
308 testCSV141Ok(CSVFormat.EXCEL);
309 }
310
311 private void testCSV141Failure(final CSVFormat format, final int failParseRecordNo) throws IOException {
312 final Path path = Paths.get("src/test/resources/org/apache/commons/csv/CSV-141/csv-141.csv");
313 try (final CSVParser parser = CSVParser.parse(path, StandardCharsets.UTF_8, format)) {
314
315 CSVRecord record = parse(parser, failParseRecordNo);
316 if (record == null) {
317 return;
318 }
319 assertEquals("1414770317901", record.get(0));
320 assertEquals("android.widget.EditText", record.get(1));
321 assertEquals("pass sem1 _84*|*", record.get(2));
322 assertEquals("0", record.get(3));
323 assertEquals("pass sem1 _8", record.get(4));
324 assertEquals(5, record.size());
325
326 record = parse(parser, failParseRecordNo);
327 if (record == null) {
328 return;
329 }
330 assertEquals("1414770318470", record.get(0));
331 assertEquals("android.widget.EditText", record.get(1));
332 assertEquals("pass sem1 _84:|", record.get(2));
333 assertEquals("0", record.get(3));
334 assertEquals("pass sem1 _84:\\", record.get(4));
335 assertEquals(5, record.size());
336
337 assertThrows(IOException.class, () -> parser.nextRecord());
338 }
339 }
340
341 private void testCSV141Ok(final CSVFormat format) throws IOException {
342 final Path path = Paths.get("src/test/resources/org/apache/commons/csv/CSV-141/csv-141.csv");
343 try (final CSVParser parser = CSVParser.parse(path, StandardCharsets.UTF_8, format)) {
344
345 CSVRecord record = parser.nextRecord();
346 assertEquals("1414770317901", record.get(0));
347 assertEquals("android.widget.EditText", record.get(1));
348 assertEquals("pass sem1 _84*|*", record.get(2));
349 assertEquals("0", record.get(3));
350 assertEquals("pass sem1 _8", record.get(4));
351 assertEquals(5, record.size());
352
353 record = parser.nextRecord();
354 assertEquals("1414770318470", record.get(0));
355 assertEquals("android.widget.EditText", record.get(1));
356 assertEquals("pass sem1 _84:|", record.get(2));
357 assertEquals("0", record.get(3));
358 assertEquals("pass sem1 _84:\\", record.get(4));
359 assertEquals(5, record.size());
360
361 record = parser.nextRecord();
362 assertEquals("1414770318327", record.get(0));
363 assertEquals("android.widget.EditText", record.get(1));
364 assertEquals("pass sem1\n1414770318628\"", record.get(2));
365 assertEquals("android.widget.EditText", record.get(3));
366 assertEquals("pass sem1 _84*|*", record.get(4));
367 assertEquals("0", record.get(5));
368 assertEquals("pass sem1\n", record.get(6));
369 assertEquals(7, record.size());
370
371 record = parser.nextRecord();
372 assertNull(record);
373 }
374 }
375
376 @Test
377 public void testCSV141RFC4180() throws Exception {
378 testCSV141Failure(CSVFormat.RFC4180, 3);
379 }
380
381 @Test
382 public void testCSV235() throws IOException {
383 final String dqString = "\"aaa\",\"b\"\"bb\",\"ccc\"";
384 try (final CSVParser parser = CSVFormat.RFC4180.parse(new StringReader(dqString))) {
385 final Iterator<CSVRecord> records = parser.iterator();
386 final CSVRecord record = records.next();
387 assertFalse(records.hasNext());
388 assertEquals(3, record.size());
389 assertEquals("aaa", record.get(0));
390 assertEquals("b\"bb", record.get(1));
391 assertEquals("ccc", record.get(2));
392 }
393 }
394
395 @Test
396 public void testCSV57() throws Exception {
397 try (final CSVParser parser = CSVParser.parse("", CSVFormat.DEFAULT)) {
398 final List<CSVRecord> list = parser.getRecords();
399 assertNotNull(list);
400 assertEquals(0, list.size());
401 }
402 }
403
404 @Test
405 public void testDefaultFormat() throws IOException {
406 final String code = "" + "a,b#\n"
407 + "\"\n\",\" \",#\n"
408 + "#,\"\"\n"
409 + "# Final comment\n"
410 ;
411 final String[][] res = {{"a", "b#"}, {"\n", " ", "#"}, {"#", ""}, {"# Final comment"}};
412
413 CSVFormat format = CSVFormat.DEFAULT;
414 assertFalse(format.isCommentMarkerSet());
415 final String[][] res_comments = {{"a", "b#"}, {"\n", " ", "#"},};
416
417 try (final CSVParser parser = CSVParser.parse(code, format)) {
418 final List<CSVRecord> records = parser.getRecords();
419 assertFalse(records.isEmpty());
420
421 Utils.compare("Failed to parse without comments", res, records);
422
423 format = CSVFormat.DEFAULT.withCommentMarker('#');
424 }
425 try (final CSVParser parser = CSVParser.parse(code, format)) {
426 final List<CSVRecord> records = parser.getRecords();
427
428 Utils.compare("Failed to parse with comments", res_comments, records);
429 }
430 }
431
432 @Test
433 public void testDuplicateHeadersAllowedByDefault() throws Exception {
434 try (CSVParser parser = CSVParser.parse("a,b,a\n1,2,3\nx,y,z", CSVFormat.DEFAULT.withHeader())) {
435
436 }
437 }
438
439 @Test
440 public void testDuplicateHeadersNotAllowed() {
441 assertThrows(IllegalArgumentException.class,
442 () -> CSVParser.parse("a,b,a\n1,2,3\nx,y,z", CSVFormat.DEFAULT.withHeader().withAllowDuplicateHeaderNames(false)));
443 }
444
445 @Test
446 public void testEmptyFile() throws Exception {
447 try (final CSVParser parser = CSVParser.parse(Paths.get("src/test/resources/org/apache/commons/csv/empty.txt"), StandardCharsets.UTF_8,
448 CSVFormat.DEFAULT)) {
449 assertNull(parser.nextRecord());
450 }
451 }
452
453 @Test
454 public void testEmptyFileHeaderParsing() throws Exception {
455 try (final CSVParser parser = CSVParser.parse("", CSVFormat.DEFAULT.withFirstRecordAsHeader())) {
456 assertNull(parser.nextRecord());
457 assertTrue(parser.getHeaderNames().isEmpty());
458 }
459 }
460
461 @Test
462 public void testEmptyLineBehaviorCSV() throws Exception {
463 final String[] codes = {"hello,\r\n\r\n\r\n", "hello,\n\n\n", "hello,\"\"\r\n\r\n\r\n", "hello,\"\"\n\n\n"};
464 final String[][] res = {{"hello", ""}
465 };
466 for (final String code : codes) {
467 try (final CSVParser parser = CSVParser.parse(code, CSVFormat.DEFAULT)) {
468 final List<CSVRecord> records = parser.getRecords();
469 assertEquals(res.length, records.size());
470 assertFalse(records.isEmpty());
471 for (int i = 0; i < res.length; i++) {
472 assertArrayEquals(res[i], records.get(i).values());
473 }
474 }
475 }
476 }
477
478 @Test
479 public void testEmptyLineBehaviorExcel() throws Exception {
480 final String[] codes = {"hello,\r\n\r\n\r\n", "hello,\n\n\n", "hello,\"\"\r\n\r\n\r\n", "hello,\"\"\n\n\n"};
481 final String[][] res = {{"hello", ""}, {""},
482 {""}};
483 for (final String code : codes) {
484 try (final CSVParser parser = CSVParser.parse(code, CSVFormat.EXCEL)) {
485 final List<CSVRecord> records = parser.getRecords();
486 assertEquals(res.length, records.size());
487 assertFalse(records.isEmpty());
488 for (int i = 0; i < res.length; i++) {
489 assertArrayEquals(res[i], records.get(i).values());
490 }
491 }
492 }
493 }
494
495 @Test
496 public void testEmptyString() throws Exception {
497 try (final CSVParser parser = CSVParser.parse("", CSVFormat.DEFAULT)) {
498 assertNull(parser.nextRecord());
499 }
500 }
501
502 @Test
503 public void testEndOfFileBehaviorCSV() throws Exception {
504 final String[] codes = {"hello,\r\n\r\nworld,\r\n", "hello,\r\n\r\nworld,", "hello,\r\n\r\nworld,\"\"\r\n", "hello,\r\n\r\nworld,\"\"",
505 "hello,\r\n\r\nworld,\n", "hello,\r\n\r\nworld,", "hello,\r\n\r\nworld,\"\"\n", "hello,\r\n\r\nworld,\"\""};
506 final String[][] res = {{"hello", ""},
507 {"world", ""}};
508 for (final String code : codes) {
509 try (final CSVParser parser = CSVParser.parse(code, CSVFormat.DEFAULT)) {
510 final List<CSVRecord> records = parser.getRecords();
511 assertEquals(res.length, records.size());
512 assertFalse(records.isEmpty());
513 for (int i = 0; i < res.length; i++) {
514 assertArrayEquals(res[i], records.get(i).values());
515 }
516 }
517 }
518 }
519
520 @Test
521 public void testEndOfFileBehaviorExcel() throws Exception {
522 final String[] codes = {"hello,\r\n\r\nworld,\r\n", "hello,\r\n\r\nworld,", "hello,\r\n\r\nworld,\"\"\r\n", "hello,\r\n\r\nworld,\"\"",
523 "hello,\r\n\r\nworld,\n", "hello,\r\n\r\nworld,", "hello,\r\n\r\nworld,\"\"\n", "hello,\r\n\r\nworld,\"\""};
524 final String[][] res = {{"hello", ""}, {""},
525 {"world", ""}};
526
527 for (final String code : codes) {
528 try (final CSVParser parser = CSVParser.parse(code, CSVFormat.EXCEL)) {
529 final List<CSVRecord> records = parser.getRecords();
530 assertEquals(res.length, records.size());
531 assertFalse(records.isEmpty());
532 for (int i = 0; i < res.length; i++) {
533 assertArrayEquals(res[i], records.get(i).values());
534 }
535 }
536 }
537 }
538
539 @Test
540 public void testExcelFormat1() throws IOException {
541 final String code = "value1,value2,value3,value4\r\na,b,c,d\r\n x,,," + "\r\n\r\n\"\"\"hello\"\"\",\" \"\"world\"\"\",\"abc\ndef\",\r\n";
542 final String[][] res = {{"value1", "value2", "value3", "value4"}, {"a", "b", "c", "d"}, {" x", "", "", ""}, {""},
543 {"\"hello\"", " \"world\"", "abc\ndef", ""}};
544 try (final CSVParser parser = CSVParser.parse(code, CSVFormat.EXCEL)) {
545 final List<CSVRecord> records = parser.getRecords();
546 assertEquals(res.length, records.size());
547 assertFalse(records.isEmpty());
548 for (int i = 0; i < res.length; i++) {
549 assertArrayEquals(res[i], records.get(i).values());
550 }
551 }
552 }
553
554 @Test
555 public void testExcelFormat2() throws Exception {
556 final String code = "foo,baar\r\n\r\nhello,\r\n\r\nworld,\r\n";
557 final String[][] res = {{"foo", "baar"}, {""}, {"hello", ""}, {""}, {"world", ""}};
558 try (final CSVParser parser = CSVParser.parse(code, CSVFormat.EXCEL)) {
559 final List<CSVRecord> records = parser.getRecords();
560 assertEquals(res.length, records.size());
561 assertFalse(records.isEmpty());
562 for (int i = 0; i < res.length; i++) {
563 assertArrayEquals(res[i], records.get(i).values());
564 }
565 }
566 }
567
568
569
570
571 @Test
572 public void testExcelHeaderCountLessThanData() throws Exception {
573 final String code = "A,B,C,,\r\na,b,c,d,e\r\n";
574 try (final CSVParser parser = CSVParser.parse(code, CSVFormat.EXCEL.withHeader())) {
575 parser.getRecords().forEach(record -> {
576 assertEquals("a", record.get("A"));
577 assertEquals("b", record.get("B"));
578 assertEquals("c", record.get("C"));
579 });
580 }
581 }
582
583 @Test
584 public void testFirstEndOfLineCr() throws IOException {
585 final String data = "foo\rbaar,\rhello,world\r,kanu";
586 try (final CSVParser parser = CSVParser.parse(data, CSVFormat.DEFAULT)) {
587 final List<CSVRecord> records = parser.getRecords();
588 assertEquals(4, records.size());
589 assertEquals("\r", parser.getFirstEndOfLine());
590 }
591 }
592
593 @Test
594 public void testFirstEndOfLineCrLf() throws IOException {
595 final String data = "foo\r\nbaar,\r\nhello,world\r\n,kanu";
596 try (final CSVParser parser = CSVParser.parse(data, CSVFormat.DEFAULT)) {
597 final List<CSVRecord> records = parser.getRecords();
598 assertEquals(4, records.size());
599 assertEquals("\r\n", parser.getFirstEndOfLine());
600 }
601 }
602
603 @Test
604 public void testFirstEndOfLineLf() throws IOException {
605 final String data = "foo\nbaar,\nhello,world\n,kanu";
606 try (final CSVParser parser = CSVParser.parse(data, CSVFormat.DEFAULT)) {
607 final List<CSVRecord> records = parser.getRecords();
608 assertEquals(4, records.size());
609 assertEquals("\n", parser.getFirstEndOfLine());
610 }
611 }
612
613 @Test
614 public void testForEach() throws Exception {
615 try (final Reader in = new StringReader("a,b,c\n1,2,3\nx,y,z"); final CSVParser parser = CSVFormat.DEFAULT.parse(in)) {
616 final List<CSVRecord> records = new ArrayList<>();
617 for (final CSVRecord record : parser) {
618 records.add(record);
619 }
620 assertEquals(3, records.size());
621 assertArrayEquals(new String[] {"a", "b", "c"}, records.get(0).values());
622 assertArrayEquals(new String[] {"1", "2", "3"}, records.get(1).values());
623 assertArrayEquals(new String[] {"x", "y", "z"}, records.get(2).values());
624 }
625 }
626
627 @Test
628 public void testGetHeaderComment_HeaderComment1() throws IOException {
629 try (CSVParser parser = CSVParser.parse(CSV_INPUT_HEADER_COMMENT, FORMAT_AUTO_HEADER)) {
630 parser.getRecords();
631
632 assertTrue(parser.hasHeaderComment());
633 assertEquals("header comment", parser.getHeaderComment());
634 }
635 }
636
637 @Test
638 public void testGetHeaderComment_HeaderComment2() throws IOException {
639 try (CSVParser parser = CSVParser.parse(CSV_INPUT_HEADER_COMMENT, FORMAT_EXPLICIT_HEADER)) {
640 parser.getRecords();
641
642 assertTrue(parser.hasHeaderComment());
643 assertEquals("header comment", parser.getHeaderComment());
644 }
645 }
646
647 @Test
648 public void testGetHeaderComment_HeaderComment3() throws IOException {
649 try (CSVParser parser = CSVParser.parse(CSV_INPUT_HEADER_COMMENT, FORMAT_EXPLICIT_HEADER_NOSKIP)) {
650 parser.getRecords();
651
652 assertFalse(parser.hasHeaderComment());
653 assertNull(parser.getHeaderComment());
654 }
655 }
656
657 @Test
658 public void testGetHeaderComment_HeaderTrailerComment() throws IOException {
659 try (CSVParser parser = CSVParser.parse(CSV_INPUT_MULTILINE_HEADER_TRAILER_COMMENT, FORMAT_AUTO_HEADER)) {
660 parser.getRecords();
661
662 assertTrue(parser.hasHeaderComment());
663 assertEquals("multi-line"+LF+"header comment", parser.getHeaderComment());
664 }
665 }
666
667 @Test
668 public void testGetHeaderComment_NoComment1() throws IOException {
669 try (CSVParser parser = CSVParser.parse(CSV_INPUT_NO_COMMENT, FORMAT_AUTO_HEADER)) {
670 parser.getRecords();
671
672 assertFalse(parser.hasHeaderComment());
673 assertNull(parser.getHeaderComment());
674 }
675 }
676
677 @Test
678 public void testGetHeaderComment_NoComment2() throws IOException {
679 try (CSVParser parser = CSVParser.parse(CSV_INPUT_NO_COMMENT, FORMAT_EXPLICIT_HEADER)) {
680 parser.getRecords();
681
682 assertFalse(parser.hasHeaderComment());
683 assertNull(parser.getHeaderComment());
684 }
685 }
686
687 @Test
688 public void testGetHeaderComment_NoComment3() throws IOException {
689 try (CSVParser parser = CSVParser.parse(CSV_INPUT_NO_COMMENT, FORMAT_EXPLICIT_HEADER_NOSKIP)) {
690 parser.getRecords();
691
692 assertFalse(parser.hasHeaderComment());
693 assertNull(parser.getHeaderComment());
694 }
695 }
696
697 @Test
698 public void testGetHeaderMap() throws Exception {
699 try (final CSVParser parser = CSVParser.parse("a,b,c\n1,2,3\nx,y,z", CSVFormat.DEFAULT.withHeader("A", "B", "C"))) {
700 final Map<String, Integer> headerMap = parser.getHeaderMap();
701 final Iterator<String> columnNames = headerMap.keySet().iterator();
702
703 assertEquals("A", columnNames.next());
704 assertEquals("B", columnNames.next());
705 assertEquals("C", columnNames.next());
706 final Iterator<CSVRecord> records = parser.iterator();
707
708
709 for (int i = 0; i < 3; i++) {
710 assertTrue(records.hasNext());
711 final CSVRecord record = records.next();
712 assertEquals(record.get(0), record.get("A"));
713 assertEquals(record.get(1), record.get("B"));
714 assertEquals(record.get(2), record.get("C"));
715 }
716
717 assertFalse(records.hasNext());
718 }
719 }
720
721 @Test
722 public void testGetHeaderNames() throws IOException {
723 try (final CSVParser parser = CSVParser.parse("a,b,c\n1,2,3\nx,y,z", CSVFormat.DEFAULT.withHeader("A", "B", "C"))) {
724 final Map<String, Integer> nameIndexMap = parser.getHeaderMap();
725 final List<String> headerNames = parser.getHeaderNames();
726 assertNotNull(headerNames);
727 assertEquals(nameIndexMap.size(), headerNames.size());
728 for (int i = 0; i < headerNames.size(); i++) {
729 final String name = headerNames.get(i);
730 assertEquals(i, nameIndexMap.get(name).intValue());
731 }
732 }
733 }
734
735 @Test
736 public void testGetHeaderNamesReadOnly() throws IOException {
737 try (final CSVParser parser = CSVParser.parse("a,b,c\n1,2,3\nx,y,z", CSVFormat.DEFAULT.withHeader("A", "B", "C"))) {
738 final List<String> headerNames = parser.getHeaderNames();
739 assertNotNull(headerNames);
740 assertThrows(UnsupportedOperationException.class, () -> headerNames.add("This is a read-only list."));
741 }
742 }
743
744 @Test
745 public void testGetLine() throws IOException {
746 try (final CSVParser parser = CSVParser.parse(CSV_INPUT, CSVFormat.DEFAULT.withIgnoreSurroundingSpaces())) {
747 for (final String[] re : RESULT) {
748 assertArrayEquals(re, parser.nextRecord().values());
749 }
750
751 assertNull(parser.nextRecord());
752 }
753 }
754
755 @Test
756 public void testGetLineNumberWithCR() throws Exception {
757 this.validateLineNumbers(String.valueOf(CR));
758 }
759
760 @Test
761 public void testGetLineNumberWithCRLF() throws Exception {
762 this.validateLineNumbers(CRLF);
763 }
764
765 @Test
766 public void testGetLineNumberWithLF() throws Exception {
767 this.validateLineNumbers(String.valueOf(LF));
768 }
769
770 @Test
771 public void testGetOneLine() throws IOException {
772 try (final CSVParser parser = CSVParser.parse(CSV_INPUT_1, CSVFormat.DEFAULT)) {
773 final CSVRecord record = parser.getRecords().get(0);
774 assertArrayEquals(RESULT[0], record.values());
775 }
776 }
777
778
779
780
781
782
783 @Test
784 public void testGetOneLineOneParser() throws IOException {
785 final CSVFormat format = CSVFormat.DEFAULT;
786 try (final PipedWriter writer = new PipedWriter(); final CSVParser parser = new CSVParser(new PipedReader(writer), format)) {
787 writer.append(CSV_INPUT_1);
788 writer.append(format.getRecordSeparator());
789 final CSVRecord record1 = parser.nextRecord();
790 assertArrayEquals(RESULT[0], record1.values());
791 writer.append(CSV_INPUT_2);
792 writer.append(format.getRecordSeparator());
793 final CSVRecord record2 = parser.nextRecord();
794 assertArrayEquals(RESULT[1], record2.values());
795 }
796 }
797
798 @Test
799 public void testGetRecordNumberWithCR() throws Exception {
800 this.validateRecordNumbers(String.valueOf(CR));
801 }
802
803 @Test
804 public void testGetRecordNumberWithCRLF() throws Exception {
805 this.validateRecordNumbers(CRLF);
806 }
807
808 @Test
809 public void testGetRecordNumberWithLF() throws Exception {
810 this.validateRecordNumbers(String.valueOf(LF));
811 }
812
813 @Test
814 public void testGetRecordPositionWithCRLF() throws Exception {
815 this.validateRecordPosition(CRLF);
816 }
817
818 @Test
819 public void testGetRecordPositionWithLF() throws Exception {
820 this.validateRecordPosition(String.valueOf(LF));
821 }
822
823 @Test
824 public void testGetRecords() throws IOException {
825 try (final CSVParser parser = CSVParser.parse(CSV_INPUT, CSVFormat.DEFAULT.withIgnoreSurroundingSpaces())) {
826 final List<CSVRecord> records = parser.getRecords();
827 assertEquals(RESULT.length, records.size());
828 assertFalse(records.isEmpty());
829 for (int i = 0; i < RESULT.length; i++) {
830 assertArrayEquals(RESULT[i], records.get(i).values());
831 }
832 }
833 }
834
835 @Test
836 public void testGetRecordsFromBrokenInputStream() throws IOException {
837 @SuppressWarnings("resource")
838 final CSVParser parser = CSVParser.parse(new BrokenInputStream(), UTF_8, CSVFormat.DEFAULT);
839 assertThrows(UncheckedIOException.class, parser::getRecords);
840
841 }
842
843 @Test
844 public void testGetRecordWithMultiLineValues() throws Exception {
845 try (final CSVParser parser = CSVParser.parse("\"a\r\n1\",\"a\r\n2\"" + CRLF + "\"b\r\n1\",\"b\r\n2\"" + CRLF + "\"c\r\n1\",\"c\r\n2\"",
846 CSVFormat.DEFAULT.withRecordSeparator(CRLF))) {
847 CSVRecord record;
848 assertEquals(0, parser.getRecordNumber());
849 assertEquals(0, parser.getCurrentLineNumber());
850 assertNotNull(record = parser.nextRecord());
851 assertEquals(3, parser.getCurrentLineNumber());
852 assertEquals(1, record.getRecordNumber());
853 assertEquals(1, parser.getRecordNumber());
854 assertNotNull(record = parser.nextRecord());
855 assertEquals(6, parser.getCurrentLineNumber());
856 assertEquals(2, record.getRecordNumber());
857 assertEquals(2, parser.getRecordNumber());
858 assertNotNull(record = parser.nextRecord());
859 assertEquals(9, parser.getCurrentLineNumber());
860 assertEquals(3, record.getRecordNumber());
861 assertEquals(3, parser.getRecordNumber());
862 assertNull(record = parser.nextRecord());
863 assertEquals(9, parser.getCurrentLineNumber());
864 assertEquals(3, parser.getRecordNumber());
865 }
866 }
867
868 @Test
869 public void testGetTrailerComment_HeaderComment1() throws IOException {
870 try (CSVParser parser = CSVParser.parse(CSV_INPUT_HEADER_COMMENT, FORMAT_AUTO_HEADER)) {
871 parser.getRecords();
872 assertFalse(parser.hasTrailerComment());
873 assertNull(parser.getTrailerComment());
874 }
875 }
876
877 @Test
878 public void testGetTrailerComment_HeaderComment2() throws IOException {
879 try (CSVParser parser = CSVParser.parse(CSV_INPUT_HEADER_COMMENT, FORMAT_EXPLICIT_HEADER)) {
880 parser.getRecords();
881 assertFalse(parser.hasTrailerComment());
882 assertNull(parser.getTrailerComment());
883 }
884 }
885
886 @Test
887 public void testGetTrailerComment_HeaderComment3() throws IOException {
888 try (CSVParser parser = CSVParser.parse(CSV_INPUT_HEADER_COMMENT, FORMAT_EXPLICIT_HEADER_NOSKIP)) {
889 parser.getRecords();
890 assertFalse(parser.hasTrailerComment());
891 assertNull(parser.getTrailerComment());
892 }
893 }
894
895 @Test
896 public void testGetTrailerComment_HeaderTrailerComment1() throws IOException {
897 try (CSVParser parser = CSVParser.parse(CSV_INPUT_HEADER_TRAILER_COMMENT, FORMAT_AUTO_HEADER)) {
898 parser.getRecords();
899 assertTrue(parser.hasTrailerComment());
900 assertEquals("comment", parser.getTrailerComment());
901 }
902 }
903
904 @Test
905 public void testGetTrailerComment_HeaderTrailerComment2() throws IOException {
906 try (CSVParser parser = CSVParser.parse(CSV_INPUT_HEADER_TRAILER_COMMENT, FORMAT_EXPLICIT_HEADER)) {
907 parser.getRecords();
908 assertTrue(parser.hasTrailerComment());
909 assertEquals("comment", parser.getTrailerComment());
910 }
911 }
912
913 @Test
914 public void testGetTrailerComment_HeaderTrailerComment3() throws IOException {
915 try (CSVParser parser = CSVParser.parse(CSV_INPUT_HEADER_TRAILER_COMMENT, FORMAT_EXPLICIT_HEADER_NOSKIP)) {
916 parser.getRecords();
917 assertTrue(parser.hasTrailerComment());
918 assertEquals("comment", parser.getTrailerComment());
919 }
920 }
921
922 @Test
923 public void testGetTrailerComment_MultilineComment() throws IOException {
924 try (CSVParser parser = CSVParser.parse(CSV_INPUT_MULTILINE_HEADER_TRAILER_COMMENT, FORMAT_AUTO_HEADER)) {
925 parser.getRecords();
926 assertTrue(parser.hasTrailerComment());
927 assertEquals("multi-line"+LF+"comment", parser.getTrailerComment());
928 }
929 }
930
931 @Test
932 public void testHeader() throws Exception {
933 final Reader in = new StringReader("a,b,c\n1,2,3\nx,y,z");
934
935 try (final CSVParser parser = CSVFormat.DEFAULT.withHeader().parse(in)) {
936 final Iterator<CSVRecord> records = parser.iterator();
937
938 for (int i = 0; i < 2; i++) {
939 assertTrue(records.hasNext());
940 final CSVRecord record = records.next();
941 assertEquals(record.get(0), record.get("a"));
942 assertEquals(record.get(1), record.get("b"));
943 assertEquals(record.get(2), record.get("c"));
944 }
945
946 assertFalse(records.hasNext());
947 }
948 }
949
950 @Test
951 public void testHeaderComment() throws Exception {
952 final Reader in = new StringReader("# comment\na,b,c\n1,2,3\nx,y,z");
953
954 try (final CSVParser parser = CSVFormat.DEFAULT.withCommentMarker('#').withHeader().parse(in)) {
955 final Iterator<CSVRecord> records = parser.iterator();
956
957 for (int i = 0; i < 2; i++) {
958 assertTrue(records.hasNext());
959 final CSVRecord record = records.next();
960 assertEquals(record.get(0), record.get("a"));
961 assertEquals(record.get(1), record.get("b"));
962 assertEquals(record.get(2), record.get("c"));
963 }
964
965 assertFalse(records.hasNext());
966 }
967 }
968
969 @Test
970 public void testHeaderMissing() throws Exception {
971 final Reader in = new StringReader("a,,c\n1,2,3\nx,y,z");
972
973 try (final CSVParser parser = CSVFormat.DEFAULT.withHeader().withAllowMissingColumnNames().parse(in)) {
974 final Iterator<CSVRecord> records = parser.iterator();
975
976 for (int i = 0; i < 2; i++) {
977 assertTrue(records.hasNext());
978 final CSVRecord record = records.next();
979 assertEquals(record.get(0), record.get("a"));
980 assertEquals(record.get(2), record.get("c"));
981 }
982
983 assertFalse(records.hasNext());
984 }
985 }
986
987 @Test
988 public void testHeaderMissingWithNull() throws Exception {
989 final Reader in = new StringReader("a,,c,,e\n1,2,3,4,5\nv,w,x,y,z");
990 try (final CSVParser parser = CSVFormat.DEFAULT.withHeader().withNullString("").withAllowMissingColumnNames().parse(in)) {
991 parser.iterator();
992 }
993 }
994
995 @Test
996 public void testHeadersMissing() throws Exception {
997 try (final Reader in = new StringReader("a,,c,,e\n1,2,3,4,5\nv,w,x,y,z");
998 final CSVParser parser = CSVFormat.DEFAULT.withHeader().withAllowMissingColumnNames().parse(in)) {
999 parser.iterator();
1000 }
1001 }
1002
1003 @Test
1004 public void testHeadersMissingException() {
1005 final Reader in = new StringReader("a,,c,,e\n1,2,3,4,5\nv,w,x,y,z");
1006 assertThrows(IllegalArgumentException.class, () -> CSVFormat.DEFAULT.withHeader().parse(in).iterator());
1007 }
1008
1009 @Test
1010 public void testHeadersMissingOneColumnException() {
1011 final Reader in = new StringReader("a,,c,d,e\n1,2,3,4,5\nv,w,x,y,z");
1012 assertThrows(IllegalArgumentException.class, () -> CSVFormat.DEFAULT.withHeader().parse(in).iterator());
1013 }
1014
1015 @Test
1016 public void testHeadersWithNullColumnName() throws IOException {
1017 final Reader in = new StringReader("header1,null,header3\n1,2,3\n4,5,6");
1018 try (final CSVParser parser = CSVFormat.DEFAULT.withHeader().withNullString("null").withAllowMissingColumnNames().parse(in)) {
1019 final Iterator<CSVRecord> records = parser.iterator();
1020 final CSVRecord record = records.next();
1021
1022 @SuppressWarnings("resource")
1023 final CSVParser recordParser = record.getParser();
1024 assertEquals(Arrays.asList("header1", "header3"), recordParser.getHeaderNames());
1025 assertEquals(2, recordParser.getHeaderMap().size());
1026 }
1027 }
1028
1029 @Test
1030 public void testIgnoreCaseHeaderMapping() throws Exception {
1031 final Reader reader = new StringReader("1,2,3");
1032 try (final CSVParser parser = CSVFormat.DEFAULT.withHeader("One", "TWO", "three").withIgnoreHeaderCase().parse(reader)) {
1033 final Iterator<CSVRecord> records = parser.iterator();
1034 final CSVRecord record = records.next();
1035 assertEquals("1", record.get("one"));
1036 assertEquals("2", record.get("two"));
1037 assertEquals("3", record.get("THREE"));
1038 }}
1039
1040 @Test
1041 public void testIgnoreEmptyLines() throws IOException {
1042 final String code = "\nfoo,baar\n\r\n,\n\n,world\r\n\n";
1043
1044
1045 try (final CSVParser parser = CSVParser.parse(code, CSVFormat.DEFAULT)) {
1046 final List<CSVRecord> records = parser.getRecords();
1047 assertEquals(3, records.size());
1048 }
1049 }
1050
1051 @Test
1052 public void testInvalidFormat() {
1053 assertThrows(IllegalArgumentException.class, () -> CSVFormat.DEFAULT.withDelimiter(CR));
1054 }
1055
1056 @Test
1057 public void testIterator() throws Exception {
1058 final Reader in = new StringReader("a,b,c\n1,2,3\nx,y,z");
1059
1060 try (final CSVParser parser = CSVFormat.DEFAULT.parse(in)) {
1061 final Iterator<CSVRecord> iterator = parser.iterator();
1062
1063 assertTrue(iterator.hasNext());
1064 assertThrows(UnsupportedOperationException.class, iterator::remove);
1065 assertArrayEquals(new String[] { "a", "b", "c" }, iterator.next().values());
1066 assertArrayEquals(new String[] { "1", "2", "3" }, iterator.next().values());
1067 assertTrue(iterator.hasNext());
1068 assertTrue(iterator.hasNext());
1069 assertTrue(iterator.hasNext());
1070 assertArrayEquals(new String[] { "x", "y", "z" }, iterator.next().values());
1071 assertFalse(iterator.hasNext());
1072
1073 assertThrows(NoSuchElementException.class, iterator::next);
1074 }}
1075
1076 @Test
1077 public void testIteratorSequenceBreaking() throws IOException {
1078 final String fiveRows = "1\n2\n3\n4\n5\n";
1079
1080
1081 try (CSVParser parser = CSVFormat.DEFAULT.parse(new StringReader(fiveRows))) {
1082
1083 final Iterator<CSVRecord> iter = parser.iterator();
1084 int recordNumber = 0;
1085 while (iter.hasNext()) {
1086 final CSVRecord record = iter.next();
1087 recordNumber++;
1088 assertEquals(String.valueOf(recordNumber), record.get(0));
1089 if (recordNumber >= 2) {
1090 break;
1091 }
1092 }
1093 iter.hasNext();
1094 while (iter.hasNext()) {
1095 final CSVRecord record = iter.next();
1096 recordNumber++;
1097 assertEquals(String.valueOf(recordNumber), record.get(0));
1098 }
1099 }
1100
1101
1102 try (CSVParser parser = CSVFormat.DEFAULT.parse(new StringReader(fiveRows))) {
1103 int recordNumber = 0;
1104 for (final CSVRecord record : parser) {
1105 recordNumber++;
1106 assertEquals(String.valueOf(recordNumber), record.get(0));
1107 if (recordNumber >= 2) {
1108 break;
1109 }
1110 }
1111 for (final CSVRecord record : parser) {
1112 recordNumber++;
1113 assertEquals(String.valueOf(recordNumber), record.get(0));
1114 }
1115 }
1116
1117
1118 try (CSVParser parser = CSVFormat.DEFAULT.parse(new StringReader(fiveRows))) {
1119 int recordNumber = 0;
1120 for (final CSVRecord record : parser) {
1121 recordNumber++;
1122 assertEquals(String.valueOf(recordNumber), record.get(0));
1123 if (recordNumber >= 2) {
1124 break;
1125 }
1126 }
1127 parser.iterator().hasNext();
1128 for (final CSVRecord record : parser) {
1129 recordNumber++;
1130 assertEquals(String.valueOf(recordNumber), record.get(0));
1131 }
1132 }
1133 }
1134
1135 @Test
1136 public void testLineFeedEndings() throws IOException {
1137 final String code = "foo\nbaar,\nhello,world\n,kanu";
1138 try (final CSVParser parser = CSVParser.parse(code, CSVFormat.DEFAULT)) {
1139 final List<CSVRecord> records = parser.getRecords();
1140 assertEquals(4, records.size());
1141 }
1142 }
1143
1144 @Test
1145 public void testMappedButNotSetAsOutlook2007ContactExport() throws Exception {
1146 final Reader in = new StringReader("a,b,c\n1,2\nx,y,z");
1147 try (final CSVParser parser = CSVFormat.DEFAULT.withHeader("A", "B", "C").withSkipHeaderRecord().parse(in)) {
1148 final Iterator<CSVRecord> records = parser.iterator();
1149 CSVRecord record;
1150
1151
1152 record = records.next();
1153 assertTrue(record.isMapped("A"));
1154 assertTrue(record.isMapped("B"));
1155 assertTrue(record.isMapped("C"));
1156 assertTrue(record.isSet("A"));
1157 assertTrue(record.isSet("B"));
1158 assertFalse(record.isSet("C"));
1159 assertEquals("1", record.get("A"));
1160 assertEquals("2", record.get("B"));
1161 assertFalse(record.isConsistent());
1162
1163
1164 record = records.next();
1165 assertTrue(record.isMapped("A"));
1166 assertTrue(record.isMapped("B"));
1167 assertTrue(record.isMapped("C"));
1168 assertTrue(record.isSet("A"));
1169 assertTrue(record.isSet("B"));
1170 assertTrue(record.isSet("C"));
1171 assertEquals("x", record.get("A"));
1172 assertEquals("y", record.get("B"));
1173 assertEquals("z", record.get("C"));
1174 assertTrue(record.isConsistent());
1175
1176 assertFalse(records.hasNext());
1177 }
1178 }
1179
1180 @Test
1181 @Disabled
1182 public void testMongoDbCsv() throws Exception {
1183 try (final CSVParser parser = CSVParser.parse("\"a a\",b,c" + LF + "d,e,f", CSVFormat.MONGODB_CSV)) {
1184 final Iterator<CSVRecord> itr1 = parser.iterator();
1185 final Iterator<CSVRecord> itr2 = parser.iterator();
1186
1187 final CSVRecord first = itr1.next();
1188 assertEquals("a a", first.get(0));
1189 assertEquals("b", first.get(1));
1190 assertEquals("c", first.get(2));
1191
1192 final CSVRecord second = itr2.next();
1193 assertEquals("d", second.get(0));
1194 assertEquals("e", second.get(1));
1195 assertEquals("f", second.get(2));
1196 }
1197 }
1198
1199 @Test
1200
1201 public void testMultipleIterators() throws Exception {
1202 try (final CSVParser parser = CSVParser.parse("a,b,c" + CRLF + "d,e,f", CSVFormat.DEFAULT)) {
1203 final Iterator<CSVRecord> itr1 = parser.iterator();
1204
1205 final CSVRecord first = itr1.next();
1206 assertEquals("a", first.get(0));
1207 assertEquals("b", first.get(1));
1208 assertEquals("c", first.get(2));
1209
1210 final CSVRecord second = itr1.next();
1211 assertEquals("d", second.get(0));
1212 assertEquals("e", second.get(1));
1213 assertEquals("f", second.get(2));
1214 }
1215 }
1216
1217 @Test
1218 public void testNewCSVParserNullReaderFormat() {
1219 assertThrows(NullPointerException.class, () -> new CSVParser(null, CSVFormat.DEFAULT));
1220 }
1221
1222 @Test
1223 public void testNewCSVParserReaderNullFormat() {
1224 assertThrows(NullPointerException.class, () -> new CSVParser(new StringReader(""), null));
1225 }
1226
1227 @Test
1228 public void testNoHeaderMap() throws Exception {
1229 try (final CSVParser parser = CSVParser.parse("a,b,c\n1,2,3\nx,y,z", CSVFormat.DEFAULT)) {
1230 assertNull(parser.getHeaderMap());
1231 }
1232 }
1233
1234 @Test
1235 public void testNotValueCSV() throws IOException {
1236 final String source = "#";
1237 final CSVFormat csvFormat = CSVFormat.DEFAULT.withCommentMarker('#');
1238 try (final CSVParser csvParser = csvFormat.parse(new StringReader(source))) {
1239 final CSVRecord csvRecord = csvParser.nextRecord();
1240 assertNull(csvRecord);
1241 }
1242 }
1243
1244 @Test
1245 public void testParse() throws Exception {
1246 final ClassLoader loader = ClassLoader.getSystemClassLoader();
1247 final URL url = loader.getResource("org/apache/commons/csv/CSVFileParser/test.csv");
1248 final CSVFormat format = CSVFormat.DEFAULT.withHeader("A", "B", "C", "D");
1249 final Charset charset = StandardCharsets.UTF_8;
1250
1251 try (@SuppressWarnings("resource")
1252 final CSVParser parser = CSVParser.parse(new InputStreamReader(url.openStream(), charset), format)) {
1253 parseFully(parser);
1254 }
1255 try (final CSVParser parser = CSVParser.parse(new String(Files.readAllBytes(Paths.get(url.toURI())), charset), format)) {
1256 parseFully(parser);
1257 }
1258 try (final CSVParser parser = CSVParser.parse(new File(url.toURI()), charset, format)) {
1259 parseFully(parser);
1260 }
1261 try (@SuppressWarnings("resource")
1262 final CSVParser parser = CSVParser.parse(url.openStream(), charset, format)) {
1263 parseFully(parser);
1264 }
1265 try (final CSVParser parser = CSVParser.parse(Paths.get(url.toURI()), charset, format)) {
1266 parseFully(parser);
1267 }
1268 try (final CSVParser parser = CSVParser.parse(url, charset, format)) {
1269 parseFully(parser);
1270 }
1271 try (final CSVParser parser = new CSVParser(new InputStreamReader(url.openStream(), charset), format)) {
1272 parseFully(parser);
1273 }
1274 try (final CSVParser parser = new CSVParser(new InputStreamReader(url.openStream(), charset), format, 0, 1)) {
1275 parseFully(parser);
1276 }
1277 }
1278
1279 @Test
1280 public void testParseFileNullFormat() {
1281 assertThrows(NullPointerException.class, () -> CSVParser.parse(new File("CSVFileParser/test.csv"), Charset.defaultCharset(), null));
1282 }
1283
1284 @Test
1285 public void testParseNullFileFormat() {
1286 assertThrows(NullPointerException.class, () -> CSVParser.parse((File) null, Charset.defaultCharset(), CSVFormat.DEFAULT));
1287 }
1288
1289 @Test
1290 public void testParseNullPathFormat() {
1291 assertThrows(NullPointerException.class, () -> CSVParser.parse((Path) null, Charset.defaultCharset(), CSVFormat.DEFAULT));
1292 }
1293
1294 @Test
1295 public void testParseNullStringFormat() {
1296 assertThrows(NullPointerException.class, () -> CSVParser.parse((String) null, CSVFormat.DEFAULT));
1297 }
1298
1299 @Test
1300 public void testParseNullUrlCharsetFormat() {
1301 assertThrows(NullPointerException.class, () -> CSVParser.parse((URL) null, Charset.defaultCharset(), CSVFormat.DEFAULT));
1302 }
1303
1304 @Test
1305 public void testParserUrlNullCharsetFormat() {
1306 assertThrows(NullPointerException.class, () -> CSVParser.parse(new URL("https://commons.apache.org"), null, CSVFormat.DEFAULT));
1307 }
1308
1309 @Test
1310 public void testParseStringNullFormat() {
1311 assertThrows(NullPointerException.class, () -> CSVParser.parse("csv data", (CSVFormat) null));
1312 }
1313
1314 @Test
1315 public void testParseUrlCharsetNullFormat() {
1316 assertThrows(NullPointerException.class, () -> CSVParser.parse(new URL("https://commons.apache.org"), Charset.defaultCharset(), null));
1317 }
1318
1319 @Test
1320 public void testParseWithDelimiterStringWithEscape() throws IOException {
1321 final String source = "a![!|!]b![|]c[|]xyz\r\nabc[abc][|]xyz";
1322 final CSVFormat csvFormat = CSVFormat.DEFAULT.builder().setDelimiter("[|]").setEscape('!').build();
1323 try (CSVParser csvParser = csvFormat.parse(new StringReader(source))) {
1324 CSVRecord csvRecord = csvParser.nextRecord();
1325 assertEquals("a[|]b![|]c", csvRecord.get(0));
1326 assertEquals("xyz", csvRecord.get(1));
1327 csvRecord = csvParser.nextRecord();
1328 assertEquals("abc[abc]", csvRecord.get(0));
1329 assertEquals("xyz", csvRecord.get(1));
1330 }
1331 }
1332
1333 @Test
1334 public void testParseWithDelimiterStringWithQuote() throws IOException {
1335 final String source = "'a[|]b[|]c'[|]xyz\r\nabc[abc][|]xyz";
1336 final CSVFormat csvFormat = CSVFormat.DEFAULT.builder().setDelimiter("[|]").setQuote('\'').build();
1337 try (CSVParser csvParser = csvFormat.parse(new StringReader(source))) {
1338 CSVRecord csvRecord = csvParser.nextRecord();
1339 assertEquals("a[|]b[|]c", csvRecord.get(0));
1340 assertEquals("xyz", csvRecord.get(1));
1341 csvRecord = csvParser.nextRecord();
1342 assertEquals("abc[abc]", csvRecord.get(0));
1343 assertEquals("xyz", csvRecord.get(1));
1344 }
1345 }
1346
1347 @Test
1348 public void testParseWithDelimiterWithEscape() throws IOException {
1349 final String source = "a!,b!,c,xyz";
1350 final CSVFormat csvFormat = CSVFormat.DEFAULT.withEscape('!');
1351 try (CSVParser csvParser = csvFormat.parse(new StringReader(source))) {
1352 final CSVRecord csvRecord = csvParser.nextRecord();
1353 assertEquals("a,b,c", csvRecord.get(0));
1354 assertEquals("xyz", csvRecord.get(1));
1355 }
1356 }
1357
1358 @Test
1359 public void testParseWithDelimiterWithQuote() throws IOException {
1360 final String source = "'a,b,c',xyz";
1361 final CSVFormat csvFormat = CSVFormat.DEFAULT.withQuote('\'');
1362 try (CSVParser csvParser = csvFormat.parse(new StringReader(source))) {
1363 final CSVRecord csvRecord = csvParser.nextRecord();
1364 assertEquals("a,b,c", csvRecord.get(0));
1365 assertEquals("xyz", csvRecord.get(1));
1366 }
1367 }
1368
1369 @Test
1370 public void testParseWithQuoteThrowsException() {
1371 final CSVFormat csvFormat = CSVFormat.DEFAULT.withQuote('\'');
1372 assertThrows(IOException.class, () -> csvFormat.parse(new StringReader("'a,b,c','")).nextRecord());
1373 assertThrows(IOException.class, () -> csvFormat.parse(new StringReader("'a,b,c'abc,xyz")).nextRecord());
1374 assertThrows(IOException.class, () -> csvFormat.parse(new StringReader("'abc'a,b,c',xyz")).nextRecord());
1375 }
1376
1377 @Test
1378 public void testParseWithQuoteWithEscape() throws IOException {
1379 final String source = "'a?,b?,c?d',xyz";
1380 final CSVFormat csvFormat = CSVFormat.DEFAULT.withQuote('\'').withEscape('?');
1381 try (CSVParser csvParser = csvFormat.parse(new StringReader(source))) {
1382 final CSVRecord csvRecord = csvParser.nextRecord();
1383 assertEquals("a,b,c?d", csvRecord.get(0));
1384 assertEquals("xyz", csvRecord.get(1));
1385 }
1386 }
1387
1388 @ParameterizedTest
1389 @EnumSource(CSVFormat.Predefined.class)
1390 public void testParsingPrintedEmptyFirstColumn(final CSVFormat.Predefined format) throws Exception {
1391 final String[][] lines = { { "a", "b" }, { "", "x" } };
1392 final StringWriter buf = new StringWriter();
1393 try (CSVPrinter printer = new CSVPrinter(buf, format.getFormat())) {
1394 printer.printRecords(Stream.of(lines));
1395 }
1396 try (CSVParser csvRecords = new CSVParser(new StringReader(buf.toString()), format.getFormat())) {
1397 for (final String[] line : lines) {
1398 assertArrayEquals(line, csvRecords.nextRecord().values());
1399 }
1400 assertNull(csvRecords.nextRecord());
1401 }
1402 }
1403
1404 @Test
1405 public void testProvidedHeader() throws Exception {
1406 final Reader in = new StringReader("a,b,c\n1,2,3\nx,y,z");
1407
1408 try (final CSVParser parser = CSVFormat.DEFAULT.withHeader("A", "B", "C").parse(in)) {
1409 final Iterator<CSVRecord> records = parser.iterator();
1410
1411 for (int i = 0; i < 3; i++) {
1412 assertTrue(records.hasNext());
1413 final CSVRecord record = records.next();
1414 assertTrue(record.isMapped("A"));
1415 assertTrue(record.isMapped("B"));
1416 assertTrue(record.isMapped("C"));
1417 assertFalse(record.isMapped("NOT MAPPED"));
1418 assertEquals(record.get(0), record.get("A"));
1419 assertEquals(record.get(1), record.get("B"));
1420 assertEquals(record.get(2), record.get("C"));
1421 }
1422
1423 assertFalse(records.hasNext());
1424 }
1425 }
1426
1427 @Test
1428 public void testProvidedHeaderAuto() throws Exception {
1429 final Reader in = new StringReader("a,b,c\n1,2,3\nx,y,z");
1430
1431 try (final CSVParser parser = CSVFormat.DEFAULT.withHeader().parse(in)) {
1432 final Iterator<CSVRecord> records = parser.iterator();
1433
1434 for (int i = 0; i < 2; i++) {
1435 assertTrue(records.hasNext());
1436 final CSVRecord record = records.next();
1437 assertTrue(record.isMapped("a"));
1438 assertTrue(record.isMapped("b"));
1439 assertTrue(record.isMapped("c"));
1440 assertFalse(record.isMapped("NOT MAPPED"));
1441 assertEquals(record.get(0), record.get("a"));
1442 assertEquals(record.get(1), record.get("b"));
1443 assertEquals(record.get(2), record.get("c"));
1444 }
1445
1446 assertFalse(records.hasNext());
1447 }
1448 }
1449
1450 @Test
1451 public void testRepeatedHeadersAreReturnedInCSVRecordHeaderNames() throws IOException {
1452 final Reader in = new StringReader("header1,header2,header1\n1,2,3\n4,5,6");
1453 try (final CSVParser parser = CSVFormat.DEFAULT.withFirstRecordAsHeader().withTrim().parse(in)) {
1454 final Iterator<CSVRecord> records = parser.iterator();
1455 final CSVRecord record = records.next();
1456 @SuppressWarnings("resource")
1457 final CSVParser recordParser = record.getParser();
1458 assertEquals(Arrays.asList("header1", "header2", "header1"), recordParser.getHeaderNames());
1459 }}
1460
1461 @Test
1462 public void testRoundtrip() throws Exception {
1463 final StringWriter out = new StringWriter();
1464 final String data = "a,b,c\r\n1,2,3\r\nx,y,z\r\n";
1465 try (final CSVPrinter printer = new CSVPrinter(out, CSVFormat.DEFAULT);
1466 final CSVParser parse = CSVParser.parse(data, CSVFormat.DEFAULT)) {
1467 for (final CSVRecord record : parse) {
1468 printer.printRecord(record);
1469 }
1470 assertEquals(data, out.toString());
1471 }
1472 }
1473
1474 @Test
1475 public void testSkipAutoHeader() throws Exception {
1476 final Reader in = new StringReader("a,b,c\n1,2,3\nx,y,z");
1477 try (final CSVParser parser = CSVFormat.DEFAULT.withHeader().parse(in)) {
1478 final Iterator<CSVRecord> records = parser.iterator();
1479 final CSVRecord record = records.next();
1480 assertEquals("1", record.get("a"));
1481 assertEquals("2", record.get("b"));
1482 assertEquals("3", record.get("c"));
1483 }
1484 }
1485
1486 @Test
1487 public void testSkipHeaderOverrideDuplicateHeaders() throws Exception {
1488 final Reader in = new StringReader("a,a,a\n1,2,3\nx,y,z");
1489 try (final CSVParser parser = CSVFormat.DEFAULT.withHeader("X", "Y", "Z").withSkipHeaderRecord().parse(in)) {
1490 final Iterator<CSVRecord> records = parser.iterator();
1491 final CSVRecord record = records.next();
1492 assertEquals("1", record.get("X"));
1493 assertEquals("2", record.get("Y"));
1494 assertEquals("3", record.get("Z"));
1495 }}
1496
1497 @Test
1498 public void testSkipSetAltHeaders() throws Exception {
1499 final Reader in = new StringReader("a,b,c\n1,2,3\nx,y,z");
1500 try (final CSVParser parser = CSVFormat.DEFAULT.withHeader("X", "Y", "Z").withSkipHeaderRecord().parse(in)) {
1501 final Iterator<CSVRecord> records = parser.iterator();
1502 final CSVRecord record = records.next();
1503 assertEquals("1", record.get("X"));
1504 assertEquals("2", record.get("Y"));
1505 assertEquals("3", record.get("Z"));
1506 }
1507 }
1508
1509 @Test
1510 public void testSkipSetHeader() throws Exception {
1511 final Reader in = new StringReader("a,b,c\n1,2,3\nx,y,z");
1512 try (final CSVParser parser = CSVFormat.DEFAULT.withHeader("a", "b", "c").withSkipHeaderRecord().parse(in)) {
1513 final Iterator<CSVRecord> records = parser.iterator();
1514 final CSVRecord record = records.next();
1515 assertEquals("1", record.get("a"));
1516 assertEquals("2", record.get("b"));
1517 assertEquals("3", record.get("c"));
1518 }
1519 }
1520
1521 @Test
1522 @Disabled
1523 public void testStartWithEmptyLinesThenHeaders() throws Exception {
1524 final String[] codes = {"\r\n\r\n\r\nhello,\r\n\r\n\r\n", "hello,\n\n\n", "hello,\"\"\r\n\r\n\r\n", "hello,\"\"\n\n\n"};
1525 final String[][] res = {{"hello", ""}, {""},
1526 {""}};
1527 for (final String code : codes) {
1528 try (final CSVParser parser = CSVParser.parse(code, CSVFormat.EXCEL)) {
1529 final List<CSVRecord> records = parser.getRecords();
1530 assertEquals(res.length, records.size());
1531 assertFalse(records.isEmpty());
1532 for (int i = 0; i < res.length; i++) {
1533 assertArrayEquals(res[i], records.get(i).values());
1534 }
1535 }
1536 }
1537 }
1538
1539 @Test
1540 public void testStream() throws Exception {
1541 final Reader in = new StringReader("a,b,c\n1,2,3\nx,y,z");
1542 try (final CSVParser parser = CSVFormat.DEFAULT.parse(in)) {
1543 final List<CSVRecord> list = parser.stream().collect(Collectors.toList());
1544 assertFalse(list.isEmpty());
1545 assertArrayEquals(new String[] { "a", "b", "c" }, list.get(0).values());
1546 assertArrayEquals(new String[] { "1", "2", "3" }, list.get(1).values());
1547 assertArrayEquals(new String[] { "x", "y", "z" }, list.get(2).values());
1548 }}
1549
1550 @Test
1551 public void testThrowExceptionWithLineAndPosition() throws IOException {
1552 final String csvContent = "col1,col2,col3,col4,col5,col6,col7,col8,col9,col10\nrec1,rec2,rec3,rec4,rec5,rec6,rec7,rec8,\"\"rec9\"\",rec10";
1553 final StringReader stringReader = new StringReader(csvContent);
1554
1555 final CSVFormat csvFormat = CSVFormat.DEFAULT.builder()
1556 .setHeader()
1557 .setSkipHeaderRecord(true)
1558 .build();
1559
1560
1561 try (CSVParser csvParser = csvFormat.parse(stringReader)) {
1562 final Exception exception = assertThrows(UncheckedIOException.class, csvParser::getRecords);
1563 assertTrue(exception.getMessage().contains("Invalid char between encapsulated token and delimiter at line: 2, position: 94"));
1564 }
1565 }
1566
1567 @Test
1568 public void testTrailingDelimiter() throws Exception {
1569 final Reader in = new StringReader("a,a,a,\n\"1\",\"2\",\"3\",\nx,y,z,");
1570 try (final CSVParser parser = CSVFormat.DEFAULT.withHeader("X", "Y", "Z").withSkipHeaderRecord().withTrailingDelimiter().parse(in)) {
1571 final Iterator<CSVRecord> records = parser.iterator();
1572 final CSVRecord record = records.next();
1573 assertEquals("1", record.get("X"));
1574 assertEquals("2", record.get("Y"));
1575 assertEquals("3", record.get("Z"));
1576 assertEquals(3, record.size());
1577 }
1578 }
1579
1580 @Test
1581 public void testTrim() throws Exception {
1582 final Reader in = new StringReader("a,a,a\n\" 1 \",\" 2 \",\" 3 \"\nx,y,z");
1583 try (final CSVParser parser = CSVFormat.DEFAULT.withHeader("X", "Y", "Z").withSkipHeaderRecord().withTrim().parse(in)) {
1584 final Iterator<CSVRecord> records = parser.iterator();
1585 final CSVRecord record = records.next();
1586 assertEquals("1", record.get("X"));
1587 assertEquals("2", record.get("Y"));
1588 assertEquals("3", record.get("Z"));
1589 assertEquals(3, record.size());
1590 }}
1591
1592 private void validateLineNumbers(final String lineSeparator) throws IOException {
1593 try (final CSVParser parser = CSVParser.parse("a" + lineSeparator + "b" + lineSeparator + "c", CSVFormat.DEFAULT.withRecordSeparator(lineSeparator))) {
1594 assertEquals(0, parser.getCurrentLineNumber());
1595 assertNotNull(parser.nextRecord());
1596 assertEquals(1, parser.getCurrentLineNumber());
1597 assertNotNull(parser.nextRecord());
1598 assertEquals(2, parser.getCurrentLineNumber());
1599 assertNotNull(parser.nextRecord());
1600
1601 assertEquals(3, parser.getCurrentLineNumber());
1602 assertNull(parser.nextRecord());
1603
1604 assertEquals(3, parser.getCurrentLineNumber());
1605 }
1606 }
1607
1608 private void validateRecordNumbers(final String lineSeparator) throws IOException {
1609 try (final CSVParser parser = CSVParser.parse("a" + lineSeparator + "b" + lineSeparator + "c", CSVFormat.DEFAULT.withRecordSeparator(lineSeparator))) {
1610 CSVRecord record;
1611 assertEquals(0, parser.getRecordNumber());
1612 assertNotNull(record = parser.nextRecord());
1613 assertEquals(1, record.getRecordNumber());
1614 assertEquals(1, parser.getRecordNumber());
1615 assertNotNull(record = parser.nextRecord());
1616 assertEquals(2, record.getRecordNumber());
1617 assertEquals(2, parser.getRecordNumber());
1618 assertNotNull(record = parser.nextRecord());
1619 assertEquals(3, record.getRecordNumber());
1620 assertEquals(3, parser.getRecordNumber());
1621 assertNull(record = parser.nextRecord());
1622 assertEquals(3, parser.getRecordNumber());
1623 }
1624 }
1625
1626 private void validateRecordPosition(final String lineSeparator) throws IOException {
1627 final String nl = lineSeparator;
1628
1629 final String code = "a,b,c" + lineSeparator + "1,2,3" + lineSeparator +
1630
1631 "'A" + nl + "A','B" + nl + "B',CC" + lineSeparator +
1632
1633
1634 "\u00c4,\u00d6,\u00dc" + lineSeparator + "EOF,EOF,EOF";
1635
1636 final CSVFormat format = CSVFormat.newFormat(',').withQuote('\'').withRecordSeparator(lineSeparator);
1637 CSVParser parser = CSVParser.parse(code, format);
1638
1639 CSVRecord record;
1640 assertEquals(0, parser.getRecordNumber());
1641
1642 assertNotNull(record = parser.nextRecord());
1643 assertEquals(1, record.getRecordNumber());
1644 assertEquals(code.indexOf('a'), record.getCharacterPosition());
1645
1646 assertNotNull(record = parser.nextRecord());
1647 assertEquals(2, record.getRecordNumber());
1648 assertEquals(code.indexOf('1'), record.getCharacterPosition());
1649
1650 assertNotNull(record = parser.nextRecord());
1651 final long positionRecord3 = record.getCharacterPosition();
1652 assertEquals(3, record.getRecordNumber());
1653 assertEquals(code.indexOf("'A"), record.getCharacterPosition());
1654 assertEquals("A" + lineSeparator + "A", record.get(0));
1655 assertEquals("B" + lineSeparator + "B", record.get(1));
1656 assertEquals("CC", record.get(2));
1657
1658 assertNotNull(record = parser.nextRecord());
1659 assertEquals(4, record.getRecordNumber());
1660 assertEquals(code.indexOf('\u00c4'), record.getCharacterPosition());
1661
1662 assertNotNull(record = parser.nextRecord());
1663 assertEquals(5, record.getRecordNumber());
1664 assertEquals(code.indexOf("EOF"), record.getCharacterPosition());
1665
1666 parser.close();
1667
1668
1669 parser = new CSVParser(new StringReader(code.substring((int) positionRecord3)), format, positionRecord3, 3);
1670
1671 assertNotNull(record = parser.nextRecord());
1672 assertEquals(3, record.getRecordNumber());
1673 assertEquals(code.indexOf("'A"), record.getCharacterPosition());
1674 assertEquals("A" + lineSeparator + "A", record.get(0));
1675 assertEquals("B" + lineSeparator + "B", record.get(1));
1676 assertEquals("CC", record.get(2));
1677
1678 assertNotNull(record = parser.nextRecord());
1679 assertEquals(4, record.getRecordNumber());
1680 assertEquals(code.indexOf('\u00c4'), record.getCharacterPosition());
1681 assertEquals("\u00c4", record.get(0));
1682
1683 parser.close();
1684 }
1685 }