View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *   https://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing,
13   * software distributed under the License is distributed on an
14   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15   * KIND, either express or implied.  See the License for the
16   * specific language governing permissions and limitations
17   * under the License.
18   */
19  
20  package org.apache.commons.csv;
21  
22  import static org.junit.jupiter.api.Assertions.assertEquals;
23  import static org.junit.jupiter.api.Assertions.assertNotNull;
24  import static org.junit.jupiter.api.Assertions.assertTrue;
25  import static org.junit.jupiter.api.Assertions.fail;
26  
27  import java.io.BufferedReader;
28  import java.io.File;
29  import java.io.FileReader;
30  import java.io.IOException;
31  import java.net.URL;
32  import java.nio.charset.Charset;
33  import java.nio.charset.StandardCharsets;
34  import java.util.Arrays;
35  import java.util.stream.Stream;
36  
37  import org.junit.jupiter.params.ParameterizedTest;
38  import org.junit.jupiter.params.provider.MethodSource;
39  
40  /**
41   * Parse tests using test files
42   */
43  public class CSVFileParserTest {
44  
45      private static final File BASE_DIR = new File("src/test/resources/org/apache/commons/csv/CSVFileParser");
46  
47      public static Stream<File> generateData() {
48          final File[] files = BASE_DIR.listFiles((dir, name) -> name.startsWith("test") && name.endsWith(".txt"));
49          return files != null ? Stream.of(files) : Stream.empty();
50      }
51  
52      private String readTestData(final BufferedReader reader) throws IOException {
53          String line;
54          do {
55              line = reader.readLine();
56          } while (line != null && line.startsWith("#"));
57          return line;
58      }
59  
60      @ParameterizedTest
61      @MethodSource("generateData")
62      public void testCSVFile(final File testFile) throws Exception {
63          try (FileReader fr = new FileReader(testFile); BufferedReader testDataReader = new BufferedReader(fr)) {
64              String line = readTestData(testDataReader);
65              assertNotNull("file must contain config line", line);
66              final String[] split = line.split(" ");
67              assertTrue(split.length >= 1, testFile.getName() + " require 1 param");
68              // first line starts with csv data file name
69              CSVFormat format = CSVFormat.newFormat(',').withQuote('"');
70              boolean checkComments = false;
71              for (int i = 1; i < split.length; i++) {
72                  final String option = split[i];
73                  final String[] optionParts = option.split("=", 2);
74                  if ("IgnoreEmpty".equalsIgnoreCase(optionParts[0])) {
75                      format = format.withIgnoreEmptyLines(Boolean.parseBoolean(optionParts[1]));
76                  } else if ("IgnoreSpaces".equalsIgnoreCase(optionParts[0])) {
77                      format = format.withIgnoreSurroundingSpaces(Boolean.parseBoolean(optionParts[1]));
78                  } else if ("CommentStart".equalsIgnoreCase(optionParts[0])) {
79                      format = format.withCommentMarker(optionParts[1].charAt(0));
80                  } else if ("CheckComments".equalsIgnoreCase(optionParts[0])) {
81                      checkComments = true;
82                  } else {
83                      fail(testFile.getName() + " unexpected option: " + option);
84                  }
85              }
86              line = readTestData(testDataReader); // get string version of format
87              assertEquals(line, format.toString(), testFile.getName() + " Expected format ");
88  
89              // Now parse the file and compare against the expected results
90              // We use a buffered reader internally so no need to create one here.
91              try (CSVParser parser = CSVParser.parse(new File(BASE_DIR, split[0]), Charset.defaultCharset(), format)) {
92                  for (final CSVRecord record : parser) {
93                      String parsed = Arrays.toString(record.values());
94                      final String comment = record.getComment();
95                      if (checkComments && comment != null) {
96                          parsed += "#" + comment.replace("\n", "\\n");
97                      }
98                      final int count = record.size();
99                      assertEquals(readTestData(testDataReader), count + ":" + parsed, testFile.getName());
100                 }
101             }
102         }
103     }
104 
105     @ParameterizedTest
106     @MethodSource("generateData")
107     public void testCSVUrl(final File testFile) throws Exception {
108         try (FileReader fr = new FileReader(testFile); BufferedReader testData = new BufferedReader(fr)) {
109             String line = readTestData(testData);
110             assertNotNull("file must contain config line", line);
111             final String[] split = line.split(" ");
112             assertTrue(split.length >= 1, testFile.getName() + " require 1 param");
113             // first line starts with csv data file name
114             CSVFormat format = CSVFormat.newFormat(',').withQuote('"');
115             boolean checkComments = false;
116             for (int i = 1; i < split.length; i++) {
117                 final String option = split[i];
118                 final String[] optionParts = option.split("=", 2);
119                 if ("IgnoreEmpty".equalsIgnoreCase(optionParts[0])) {
120                     format = format.withIgnoreEmptyLines(Boolean.parseBoolean(optionParts[1]));
121                 } else if ("IgnoreSpaces".equalsIgnoreCase(optionParts[0])) {
122                     format = format.withIgnoreSurroundingSpaces(Boolean.parseBoolean(optionParts[1]));
123                 } else if ("CommentStart".equalsIgnoreCase(optionParts[0])) {
124                     format = format.withCommentMarker(optionParts[1].charAt(0));
125                 } else if ("CheckComments".equalsIgnoreCase(optionParts[0])) {
126                     checkComments = true;
127                 } else {
128                     fail(testFile.getName() + " unexpected option: " + option);
129                 }
130             }
131             line = readTestData(testData); // get string version of format
132             assertEquals(line, format.toString(), testFile.getName() + " Expected format ");
133 
134             // Now parse the file and compare against the expected results
135             final URL resource = ClassLoader.getSystemResource("org/apache/commons/csv/CSVFileParser/" + split[0]);
136             try (CSVParser parser = CSVParser.parse(resource, StandardCharsets.UTF_8, format)) {
137                 for (final CSVRecord record : parser) {
138                     String parsed = Arrays.toString(record.values());
139                     final String comment = record.getComment();
140                     if (checkComments && comment != null) {
141                         parsed += "#" + comment.replace("\n", "\\n");
142                     }
143                     final int count = record.size();
144                     assertEquals(readTestData(testData), count + ":" + parsed, testFile.getName());
145                 }
146             }
147         }
148     }
149 }