View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements.  See the NOTICE file distributed with
4    * this work for additional information regarding copyright ownership.
5    * The ASF licenses this file to You under the Apache License, Version 2.0
6    * (the "License"); you may not use this file except in compliance with
7    * the License.  You may obtain a copy of the License at
8    *
9    *      http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS,
13   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14   * See the License for the specific language governing permissions and
15   * limitations under the License.
16   */
17  
18  package org.apache.commons.csv.perf;
19  
20  import java.io.BufferedReader;
21  import java.io.File;
22  import java.io.FileNotFoundException;
23  import java.io.FileOutputStream;
24  import java.io.FileReader;
25  import java.io.IOException;
26  import java.io.InputStream;
27  import java.io.OutputStream;
28  import java.io.Reader;
29  import java.util.zip.GZIPInputStream;
30  
31  import org.apache.commons.csv.CSVFormat;
32  import org.apache.commons.csv.CSVParser;
33  import org.apache.commons.csv.CSVRecord;
34  import org.apache.commons.io.IOUtils;
35  import org.junit.jupiter.api.BeforeAll;
36  import org.junit.jupiter.api.Test;
37  
38  /**
39   * Tests performance.
40   *
41   * To run this test, use: mvn test -Dtest=PerformanceTest
42   */
43  @SuppressWarnings("boxing") // test code
44  public class PerformanceTest {
45  
46      private static final String TEST_RESRC = "org/apache/commons/csv/perf/worldcitiespop.txt.gz";
47  
48      private static final File BIG_FILE = new File(System.getProperty("java.io.tmpdir"), "worldcitiespop.txt");
49      @BeforeAll
50      public static void setUpClass() throws FileNotFoundException, IOException {
51          if (BIG_FILE.exists()) {
52              System.out.println(String.format("Found test fixture %s: %,d bytes.", BIG_FILE, BIG_FILE.length()));
53              return;
54          }
55          System.out.println("Decompressing test fixture to: " + BIG_FILE + "...");
56          try (
57              final InputStream input = new GZIPInputStream(
58                  PerformanceTest.class.getClassLoader().getResourceAsStream(TEST_RESRC));
59              final OutputStream output = new FileOutputStream(BIG_FILE)) {
60              IOUtils.copy(input, output);
61              System.out.println(String.format("Decompressed test fixture %s: %,d bytes.", BIG_FILE, BIG_FILE.length()));
62          }
63      }
64  
65      private final int max = 10;
66  
67      private BufferedReader createBufferedReader() throws IOException {
68          return new BufferedReader(new FileReader(BIG_FILE));
69      }
70  
71      private long parse(final Reader reader, final boolean traverseColumns) throws IOException {
72          final CSVFormat format = CSVFormat.DEFAULT.builder().setIgnoreSurroundingSpaces(false).build();
73          long recordCount = 0;
74          try (final CSVParser parser = format.parse(reader)) {
75              for (final CSVRecord record : parser) {
76                  recordCount++;
77                  if (traverseColumns) {
78                      for (@SuppressWarnings("unused")
79                      final String value : record) {
80                          // do nothing for now
81                      }
82                  }
83              }
84          }
85          return recordCount;
86      }
87  
88      private void println(final String s) {
89          System.out.println(s);
90      }
91  
92      private long readAll(final BufferedReader in) throws IOException {
93          long count = 0;
94          while (in.readLine() != null) {
95              count++;
96          }
97          return count;
98      }
99  
100     public long testParseBigFile(final boolean traverseColumns) throws Exception {
101         final long startMillis = System.currentTimeMillis();
102         try (final BufferedReader reader = this.createBufferedReader()) {
103             final long count = this.parse(reader, traverseColumns);
104             final long totalMillis = System.currentTimeMillis() - startMillis;
105             this.println(
106                 String.format("File parsed in %,d milliseconds with Commons CSV: %,d lines.", totalMillis, count));
107             return totalMillis;
108         }
109     }
110 
111     @Test
112     public void testParseBigFileRepeat() throws Exception {
113         long bestTime = Long.MAX_VALUE;
114         for (int i = 0; i < this.max; i++) {
115             bestTime = Math.min(this.testParseBigFile(false), bestTime);
116         }
117         this.println(String.format("Best time out of %,d is %,d milliseconds.", this.max, bestTime));
118     }
119 
120     @Test
121     public void testReadBigFile() throws Exception {
122         long bestTime = Long.MAX_VALUE;
123         long count;
124         for (int i = 0; i < this.max; i++) {
125             final long startMillis;
126             try (final BufferedReader in = this.createBufferedReader()) {
127                 startMillis = System.currentTimeMillis();
128                 count = this.readAll(in);
129             }
130             final long totalMillis = System.currentTimeMillis() - startMillis;
131             bestTime = Math.min(totalMillis, bestTime);
132             this.println(String.format("File read in %,d milliseconds: %,d lines.", totalMillis, count));
133         }
134         this.println(String.format("Best time out of %,d is %,d milliseconds.", this.max, bestTime));
135     }
136 }