1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.commons.csv.perf;
19
20 import java.io.BufferedReader;
21 import java.io.File;
22 import java.io.FileNotFoundException;
23 import java.io.FileOutputStream;
24 import java.io.FileReader;
25 import java.io.IOException;
26 import java.io.InputStream;
27 import java.io.OutputStream;
28 import java.io.Reader;
29 import java.util.zip.GZIPInputStream;
30
31 import org.apache.commons.csv.CSVFormat;
32 import org.apache.commons.csv.CSVParser;
33 import org.apache.commons.csv.CSVRecord;
34 import org.apache.commons.io.IOUtils;
35 import org.junit.jupiter.api.BeforeAll;
36 import org.junit.jupiter.api.Test;
37
38
39
40
41
42
43 @SuppressWarnings("boxing")
44 public class PerformanceTest {
45
46 private static final String TEST_RESRC = "org/apache/commons/csv/perf/worldcitiespop.txt.gz";
47
48 private static final File BIG_FILE = new File(System.getProperty("java.io.tmpdir"), "worldcitiespop.txt");
49 @BeforeAll
50 public static void setUpClass() throws FileNotFoundException, IOException {
51 if (BIG_FILE.exists()) {
52 System.out.println(String.format("Found test fixture %s: %,d bytes.", BIG_FILE, BIG_FILE.length()));
53 return;
54 }
55 System.out.println("Decompressing test fixture to: " + BIG_FILE + "...");
56 try (
57 final InputStream input = new GZIPInputStream(
58 PerformanceTest.class.getClassLoader().getResourceAsStream(TEST_RESRC));
59 final OutputStream output = new FileOutputStream(BIG_FILE)) {
60 IOUtils.copy(input, output);
61 System.out.println(String.format("Decompressed test fixture %s: %,d bytes.", BIG_FILE, BIG_FILE.length()));
62 }
63 }
64
65 private final int max = 10;
66
67 private BufferedReader createBufferedReader() throws IOException {
68 return new BufferedReader(new FileReader(BIG_FILE));
69 }
70
71 private long parse(final Reader reader, final boolean traverseColumns) throws IOException {
72 final CSVFormat format = CSVFormat.DEFAULT.builder().setIgnoreSurroundingSpaces(false).build();
73 long recordCount = 0;
74 try (final CSVParser parser = format.parse(reader)) {
75 for (final CSVRecord record : parser) {
76 recordCount++;
77 if (traverseColumns) {
78 for (@SuppressWarnings("unused")
79 final String value : record) {
80
81 }
82 }
83 }
84 }
85 return recordCount;
86 }
87
88 private void println(final String s) {
89 System.out.println(s);
90 }
91
92 private long readAll(final BufferedReader in) throws IOException {
93 long count = 0;
94 while (in.readLine() != null) {
95 count++;
96 }
97 return count;
98 }
99
100 public long testParseBigFile(final boolean traverseColumns) throws Exception {
101 final long startMillis = System.currentTimeMillis();
102 try (final BufferedReader reader = this.createBufferedReader()) {
103 final long count = this.parse(reader, traverseColumns);
104 final long totalMillis = System.currentTimeMillis() - startMillis;
105 this.println(
106 String.format("File parsed in %,d milliseconds with Commons CSV: %,d lines.", totalMillis, count));
107 return totalMillis;
108 }
109 }
110
111 @Test
112 public void testParseBigFileRepeat() throws Exception {
113 long bestTime = Long.MAX_VALUE;
114 for (int i = 0; i < this.max; i++) {
115 bestTime = Math.min(this.testParseBigFile(false), bestTime);
116 }
117 this.println(String.format("Best time out of %,d is %,d milliseconds.", this.max, bestTime));
118 }
119
120 @Test
121 public void testReadBigFile() throws Exception {
122 long bestTime = Long.MAX_VALUE;
123 long count;
124 for (int i = 0; i < this.max; i++) {
125 final long startMillis;
126 try (final BufferedReader in = this.createBufferedReader()) {
127 startMillis = System.currentTimeMillis();
128 count = this.readAll(in);
129 }
130 final long totalMillis = System.currentTimeMillis() - startMillis;
131 bestTime = Math.min(totalMillis, bestTime);
132 this.println(String.format("File read in %,d milliseconds: %,d lines.", totalMillis, count));
133 }
134 this.println(String.format("Best time out of %,d is %,d milliseconds.", this.max, bestTime));
135 }
136 }