1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.commons.text;
19
20 import static org.junit.jupiter.api.Assertions.assertEquals;
21 import static org.junit.jupiter.api.Assertions.assertFalse;
22 import static org.junit.jupiter.api.Assertions.assertNotSame;
23 import static org.junit.jupiter.api.Assertions.assertNull;
24 import static org.junit.jupiter.api.Assertions.assertThrows;
25 import static org.junit.jupiter.api.Assertions.assertTrue;
26
27 import java.util.Arrays;
28 import java.util.Collections;
29 import java.util.List;
30 import java.util.NoSuchElementException;
31
32 import org.apache.commons.lang3.ArrayUtils;
33 import org.junit.jupiter.api.Test;
34
35
36
37
38
39
40 class StrTokenizerTest {
41
42 private static final String CSV_SIMPLE_FIXTURE = "A,b,c";
43
44 private static final String TSV_SIMPLE_FIXTURE = "A\tb\tc";
45
46 private void checkClone(final StrTokenizer tokenizer) {
47 assertNotSame(StrTokenizer.getCSVInstance(), tokenizer);
48 assertNotSame(StrTokenizer.getTSVInstance(), tokenizer);
49 }
50
51 @Test
52 void test1() {
53
54 final String input = "a;b;c;\"d;\"\"e\";f; ; ; ";
55 final StrTokenizer tok = new StrTokenizer(input);
56 tok.setDelimiterChar(';');
57 tok.setQuoteChar('"');
58 tok.setIgnoredMatcher(StrMatcher.trimMatcher());
59 tok.setIgnoreEmptyTokens(false);
60 final String[] tokens = tok.getTokenArray();
61
62 final String[] expected = {"a", "b", "c", "d;\"e", "f", "", "", ""};
63
64 assertEquals(expected.length, tokens.length, Arrays.toString(tokens));
65 for (int i = 0; i < expected.length; i++) {
66 assertEquals(expected[i], tokens[i],
67 "token[" + i + "] was '" + tokens[i] + "' but was expected to be '" + expected[i] + "'");
68 }
69
70 }
71
72 @Test
73 void test2() {
74
75 final String input = "a;b;c ;\"d;\"\"e\";f; ; ;";
76 final StrTokenizer tok = new StrTokenizer(input);
77 tok.setDelimiterChar(';');
78 tok.setQuoteChar('"');
79 tok.setIgnoredMatcher(StrMatcher.noneMatcher());
80 tok.setIgnoreEmptyTokens(false);
81 final String[] tokens = tok.getTokenArray();
82
83 final String[] expected = {"a", "b", "c ", "d;\"e", "f", " ", " ", ""};
84
85 assertEquals(expected.length, tokens.length, Arrays.toString(tokens));
86 for (int i = 0; i < expected.length; i++) {
87 assertEquals(expected[i], tokens[i],
88 "token[" + i + "] was '" + tokens[i] + "' but was expected to be '" + expected[i] + "'");
89 }
90
91 }
92
93 @Test
94 void test3() {
95
96 final String input = "a;b; c;\"d;\"\"e\";f; ; ;";
97 final StrTokenizer tok = new StrTokenizer(input);
98 tok.setDelimiterChar(';');
99 tok.setQuoteChar('"');
100 tok.setIgnoredMatcher(StrMatcher.noneMatcher());
101 tok.setIgnoreEmptyTokens(false);
102 final String[] tokens = tok.getTokenArray();
103
104 final String[] expected = {"a", "b", " c", "d;\"e", "f", " ", " ", ""};
105
106 assertEquals(expected.length, tokens.length, Arrays.toString(tokens));
107 for (int i = 0; i < expected.length; i++) {
108 assertEquals(expected[i], tokens[i],
109 "token[" + i + "] was '" + tokens[i] + "' but was expected to be '" + expected[i] + "'");
110 }
111
112 }
113
114 @Test
115 void test4() {
116
117 final String input = "a;b; c;\"d;\"\"e\";f; ; ;";
118 final StrTokenizer tok = new StrTokenizer(input);
119 tok.setDelimiterChar(';');
120 tok.setQuoteChar('"');
121 tok.setIgnoredMatcher(StrMatcher.trimMatcher());
122 tok.setIgnoreEmptyTokens(true);
123 final String[] tokens = tok.getTokenArray();
124
125 final String[] expected = {"a", "b", "c", "d;\"e", "f"};
126
127 assertEquals(expected.length, tokens.length, Arrays.toString(tokens));
128 for (int i = 0; i < expected.length; i++) {
129 assertEquals(expected[i], tokens[i],
130 "token[" + i + "] was '" + tokens[i] + "' but was expected to be '" + expected[i] + "'");
131 }
132
133 }
134
135 @Test
136 void test5() {
137
138 final String input = "a;b; c;\"d;\"\"e\";f; ; ;";
139 final StrTokenizer tok = new StrTokenizer(input);
140 tok.setDelimiterChar(';');
141 tok.setQuoteChar('"');
142 tok.setIgnoredMatcher(StrMatcher.trimMatcher());
143 tok.setIgnoreEmptyTokens(false);
144 tok.setEmptyTokenAsNull(true);
145 final String[] tokens = tok.getTokenArray();
146
147 final String[] expected = {"a", "b", "c", "d;\"e", "f", null, null, null};
148
149 assertEquals(expected.length, tokens.length, Arrays.toString(tokens));
150 for (int i = 0; i < expected.length; i++) {
151 assertEquals(expected[i], tokens[i],
152 "token[" + i + "] was '" + tokens[i] + "' but was expected to be '" + expected[i] + "'");
153 }
154
155 }
156
157 @Test
158 void test6() {
159
160 final String input = "a;b; c;\"d;\"\"e\";f; ; ;";
161 final StrTokenizer tok = new StrTokenizer(input);
162 tok.setDelimiterChar(';');
163 tok.setQuoteChar('"');
164 tok.setIgnoredMatcher(StrMatcher.trimMatcher());
165 tok.setIgnoreEmptyTokens(false);
166
167 final String[] tokens = tok.getTokenArray();
168
169 final String[] expected = {"a", "b", " c", "d;\"e", "f", null, null, null};
170
171 int nextCount = 0;
172 while (tok.hasNext()) {
173 tok.next();
174 nextCount++;
175 }
176
177 int prevCount = 0;
178 while (tok.hasPrevious()) {
179 tok.previous();
180 prevCount++;
181 }
182
183 assertEquals(expected.length, tokens.length, Arrays.toString(tokens));
184 assertEquals(nextCount, expected.length, "could not cycle through entire token list using the 'hasNext' and 'next' methods");
185 assertEquals(prevCount, expected.length, "could not cycle through entire token list using the 'hasPrevious' and 'previous' methods");
186 }
187
188 @Test
189 void test7() {
190
191 final String input = "a b c \"d e\" f ";
192 final StrTokenizer tok = new StrTokenizer(input);
193 tok.setDelimiterMatcher(StrMatcher.spaceMatcher());
194 tok.setQuoteMatcher(StrMatcher.doubleQuoteMatcher());
195 tok.setIgnoredMatcher(StrMatcher.noneMatcher());
196 tok.setIgnoreEmptyTokens(false);
197 final String[] tokens = tok.getTokenArray();
198
199 final String[] expected = {"a", "", "", "b", "c", "d e", "f", ""};
200
201 assertEquals(expected.length, tokens.length, Arrays.toString(tokens));
202 for (int i = 0; i < expected.length; i++) {
203 assertEquals(expected[i], tokens[i],
204 "token[" + i + "] was '" + tokens[i] + "' but was expected to be '" + expected[i] + "'");
205 }
206
207 }
208
209 @Test
210 void test8() {
211
212 final String input = "a b c \"d e\" f ";
213 final StrTokenizer tok = new StrTokenizer(input);
214 tok.setDelimiterMatcher(StrMatcher.spaceMatcher());
215 tok.setQuoteMatcher(StrMatcher.doubleQuoteMatcher());
216 tok.setIgnoredMatcher(StrMatcher.noneMatcher());
217 tok.setIgnoreEmptyTokens(true);
218 final String[] tokens = tok.getTokenArray();
219
220 final String[] expected = {"a", "b", "c", "d e", "f"};
221
222 assertEquals(expected.length, tokens.length, Arrays.toString(tokens));
223 for (int i = 0; i < expected.length; i++) {
224 assertEquals(expected[i], tokens[i],
225 "token[" + i + "] was '" + tokens[i] + "' but was expected to be '" + expected[i] + "'");
226 }
227
228 }
229
230 @Test
231 void testBasic1() {
232 final String input = "a b c";
233 final StrTokenizer tok = new StrTokenizer(input);
234 assertEquals("a", tok.next());
235 assertEquals("b", tok.next());
236 assertEquals("c", tok.next());
237 assertFalse(tok.hasNext());
238 }
239
240 @Test
241 void testBasic2() {
242 final String input = "a \nb\fc";
243 final StrTokenizer tok = new StrTokenizer(input);
244 assertEquals("a", tok.next());
245 assertEquals("b", tok.next());
246 assertEquals("c", tok.next());
247 assertFalse(tok.hasNext());
248 }
249
250 @Test
251 void testBasic3() {
252 final String input = "a \nb\u0001\fc";
253 final StrTokenizer tok = new StrTokenizer(input);
254 assertEquals("a", tok.next());
255 assertEquals("b\u0001", tok.next());
256 assertEquals("c", tok.next());
257 assertFalse(tok.hasNext());
258 }
259
260 @Test
261 void testBasic4() {
262 final String input = "a \"b\" c";
263 final StrTokenizer tok = new StrTokenizer(input);
264 assertEquals("a", tok.next());
265 assertEquals("\"b\"", tok.next());
266 assertEquals("c", tok.next());
267 assertFalse(tok.hasNext());
268 }
269
270 @Test
271 void testBasic5() {
272 final String input = "a:b':c";
273 final StrTokenizer tok = new StrTokenizer(input, ':', '\'');
274 assertEquals("a", tok.next());
275 assertEquals("b'", tok.next());
276 assertEquals("c", tok.next());
277 assertFalse(tok.hasNext());
278 }
279
280 @Test
281 void testBasicDelim1() {
282 final String input = "a:b:c";
283 final StrTokenizer tok = new StrTokenizer(input, ':');
284 assertEquals("a", tok.next());
285 assertEquals("b", tok.next());
286 assertEquals("c", tok.next());
287 assertFalse(tok.hasNext());
288 }
289
290 @Test
291 void testBasicDelim2() {
292 final String input = "a:b:c";
293 final StrTokenizer tok = new StrTokenizer(input, ',');
294 assertEquals("a:b:c", tok.next());
295 assertFalse(tok.hasNext());
296 }
297
298 @Test
299 void testBasicEmpty1() {
300 final String input = "a b c";
301 final StrTokenizer tok = new StrTokenizer(input);
302 tok.setIgnoreEmptyTokens(false);
303 assertEquals("a", tok.next());
304 assertEquals("", tok.next());
305 assertEquals("b", tok.next());
306 assertEquals("c", tok.next());
307 assertFalse(tok.hasNext());
308 }
309
310 @Test
311 void testBasicEmpty2() {
312 final String input = "a b c";
313 final StrTokenizer tok = new StrTokenizer(input);
314 tok.setIgnoreEmptyTokens(false);
315 tok.setEmptyTokenAsNull(true);
316 assertEquals("a", tok.next());
317 assertNull(tok.next());
318 assertEquals("b", tok.next());
319 assertEquals("c", tok.next());
320 assertFalse(tok.hasNext());
321 }
322
323 @Test
324 void testBasicIgnoreTrimmed1() {
325 final String input = "a: bIGNOREc : ";
326 final StrTokenizer tok = new StrTokenizer(input, ':');
327 tok.setIgnoredMatcher(StrMatcher.stringMatcher("IGNORE"));
328 tok.setTrimmerMatcher(StrMatcher.trimMatcher());
329 tok.setIgnoreEmptyTokens(false);
330 tok.setEmptyTokenAsNull(true);
331 assertEquals("a", tok.next());
332 assertEquals("bc", tok.next());
333 assertNull(tok.next());
334 assertFalse(tok.hasNext());
335 }
336
337 @Test
338 void testBasicIgnoreTrimmed2() {
339 final String input = "IGNOREaIGNORE: IGNORE bIGNOREc IGNORE : IGNORE ";
340 final StrTokenizer tok = new StrTokenizer(input, ':');
341 tok.setIgnoredMatcher(StrMatcher.stringMatcher("IGNORE"));
342 tok.setTrimmerMatcher(StrMatcher.trimMatcher());
343 tok.setIgnoreEmptyTokens(false);
344 tok.setEmptyTokenAsNull(true);
345 assertEquals("a", tok.next());
346 assertEquals("bc", tok.next());
347 assertNull(tok.next());
348 assertFalse(tok.hasNext());
349 }
350
351 @Test
352 void testBasicIgnoreTrimmed3() {
353 final String input = "IGNOREaIGNORE: IGNORE bIGNOREc IGNORE : IGNORE ";
354 final StrTokenizer tok = new StrTokenizer(input, ':');
355 tok.setIgnoredMatcher(StrMatcher.stringMatcher("IGNORE"));
356 tok.setIgnoreEmptyTokens(false);
357 tok.setEmptyTokenAsNull(true);
358 assertEquals("a", tok.next());
359 assertEquals(" bc ", tok.next());
360 assertEquals(" ", tok.next());
361 assertFalse(tok.hasNext());
362 }
363
364 @Test
365 void testBasicIgnoreTrimmed4() {
366 final String input = "IGNOREaIGNORE: IGNORE 'bIGNOREc'IGNORE'd' IGNORE : IGNORE ";
367 final StrTokenizer tok = new StrTokenizer(input, ':', '\'');
368 tok.setIgnoredMatcher(StrMatcher.stringMatcher("IGNORE"));
369 tok.setTrimmerMatcher(StrMatcher.trimMatcher());
370 tok.setIgnoreEmptyTokens(false);
371 tok.setEmptyTokenAsNull(true);
372 assertEquals("a", tok.next());
373 assertEquals("bIGNOREcd", tok.next());
374 assertNull(tok.next());
375 assertFalse(tok.hasNext());
376 }
377
378 @Test
379 void testBasicQuoted1() {
380 final String input = "a 'b' c";
381 final StrTokenizer tok = new StrTokenizer(input, ' ', '\'');
382 assertEquals("a", tok.next());
383 assertEquals("b", tok.next());
384 assertEquals("c", tok.next());
385 assertFalse(tok.hasNext());
386 }
387
388 @Test
389 void testBasicQuoted2() {
390 final String input = "a:'b':";
391 final StrTokenizer tok = new StrTokenizer(input, ':', '\'');
392 tok.setIgnoreEmptyTokens(false);
393 tok.setEmptyTokenAsNull(true);
394 assertEquals("a", tok.next());
395 assertEquals("b", tok.next());
396 assertNull(tok.next());
397 assertFalse(tok.hasNext());
398 }
399
400 @Test
401 void testBasicQuoted3() {
402 final String input = "a:'b''c'";
403 final StrTokenizer tok = new StrTokenizer(input, ':', '\'');
404 tok.setIgnoreEmptyTokens(false);
405 tok.setEmptyTokenAsNull(true);
406 assertEquals("a", tok.next());
407 assertEquals("b'c", tok.next());
408 assertFalse(tok.hasNext());
409 }
410
411 @Test
412 void testBasicQuoted4() {
413 final String input = "a: 'b' 'c' :d";
414 final StrTokenizer tok = new StrTokenizer(input, ':', '\'');
415 tok.setTrimmerMatcher(StrMatcher.trimMatcher());
416 tok.setIgnoreEmptyTokens(false);
417 tok.setEmptyTokenAsNull(true);
418 assertEquals("a", tok.next());
419 assertEquals("b c", tok.next());
420 assertEquals("d", tok.next());
421 assertFalse(tok.hasNext());
422 }
423
424 @Test
425 void testBasicQuoted5() {
426 final String input = "a: 'b'x'c' :d";
427 final StrTokenizer tok = new StrTokenizer(input, ':', '\'');
428 tok.setTrimmerMatcher(StrMatcher.trimMatcher());
429 tok.setIgnoreEmptyTokens(false);
430 tok.setEmptyTokenAsNull(true);
431 assertEquals("a", tok.next());
432 assertEquals("bxc", tok.next());
433 assertEquals("d", tok.next());
434 assertFalse(tok.hasNext());
435 }
436
437 @Test
438 void testBasicQuoted6() {
439 final String input = "a:'b'\"c':d";
440 final StrTokenizer tok = new StrTokenizer(input, ':');
441 tok.setQuoteMatcher(StrMatcher.quoteMatcher());
442 assertEquals("a", tok.next());
443 assertEquals("b\"c:d", tok.next());
444 assertFalse(tok.hasNext());
445 }
446
447 @Test
448 void testBasicQuoted7() {
449 final String input = "a:\"There's a reason here\":b";
450 final StrTokenizer tok = new StrTokenizer(input, ':');
451 tok.setQuoteMatcher(StrMatcher.quoteMatcher());
452 assertEquals("a", tok.next());
453 assertEquals("There's a reason here", tok.next());
454 assertEquals("b", tok.next());
455 assertFalse(tok.hasNext());
456 }
457
458 @Test
459 void testBasicQuotedTrimmed1() {
460 final String input = "a: 'b' :";
461 final StrTokenizer tok = new StrTokenizer(input, ':', '\'');
462 tok.setTrimmerMatcher(StrMatcher.trimMatcher());
463 tok.setIgnoreEmptyTokens(false);
464 tok.setEmptyTokenAsNull(true);
465 assertEquals("a", tok.next());
466 assertEquals("b", tok.next());
467 assertNull(tok.next());
468 assertFalse(tok.hasNext());
469 }
470
471 @Test
472 void testBasicTrimmed1() {
473 final String input = "a: b : ";
474 final StrTokenizer tok = new StrTokenizer(input, ':');
475 tok.setTrimmerMatcher(StrMatcher.trimMatcher());
476 tok.setIgnoreEmptyTokens(false);
477 tok.setEmptyTokenAsNull(true);
478 assertEquals("a", tok.next());
479 assertEquals("b", tok.next());
480 assertNull(tok.next());
481 assertFalse(tok.hasNext());
482 }
483
484 @Test
485 void testBasicTrimmed2() {
486 final String input = "a: b :";
487 final StrTokenizer tok = new StrTokenizer(input, ':');
488 tok.setTrimmerMatcher(StrMatcher.stringMatcher(" "));
489 tok.setIgnoreEmptyTokens(false);
490 tok.setEmptyTokenAsNull(true);
491 assertEquals("a", tok.next());
492 assertEquals("b", tok.next());
493 assertNull(tok.next());
494 assertFalse(tok.hasNext());
495 }
496
497 @Test
498 void testChaining() {
499 final StrTokenizer tok = new StrTokenizer();
500 assertEquals(tok, tok.reset());
501 assertEquals(tok, tok.reset(""));
502 assertEquals(tok, tok.reset(ArrayUtils.EMPTY_CHAR_ARRAY));
503 assertEquals(tok, tok.setDelimiterChar(' '));
504 assertEquals(tok, tok.setDelimiterString(" "));
505 assertEquals(tok, tok.setDelimiterMatcher(null));
506 assertEquals(tok, tok.setQuoteChar(' '));
507 assertEquals(tok, tok.setQuoteMatcher(null));
508 assertEquals(tok, tok.setIgnoredChar(' '));
509 assertEquals(tok, tok.setIgnoredMatcher(null));
510 assertEquals(tok, tok.setTrimmerMatcher(null));
511 assertEquals(tok, tok.setEmptyTokenAsNull(false));
512 assertEquals(tok, tok.setIgnoreEmptyTokens(false));
513 }
514
515
516
517
518
519 @Test
520 void testCloneNotSupportedException() {
521 final Object notCloned = new StrTokenizer() {
522
523 @Override
524 Object cloneReset() throws CloneNotSupportedException {
525 throw new CloneNotSupportedException("test");
526 }
527 }.clone();
528 assertNull(notCloned);
529 }
530
531 @Test
532 void testCloneNull() {
533 final StrTokenizer tokenizer = new StrTokenizer((char[]) null);
534
535 assertNull(tokenizer.nextToken());
536 tokenizer.reset();
537 assertNull(tokenizer.nextToken());
538
539 final StrTokenizer clonedTokenizer = (StrTokenizer) tokenizer.clone();
540 tokenizer.reset();
541 assertNull(tokenizer.nextToken());
542 assertNull(clonedTokenizer.nextToken());
543 }
544
545 @Test
546 void testCloneReset() {
547 final char[] input = {'a'};
548 final StrTokenizer tokenizer = new StrTokenizer(input);
549
550 assertEquals("a", tokenizer.nextToken());
551 tokenizer.reset(input);
552 assertEquals("a", tokenizer.nextToken());
553
554 final StrTokenizer clonedTokenizer = (StrTokenizer) tokenizer.clone();
555 input[0] = 'b';
556 tokenizer.reset(input);
557 assertEquals("b", tokenizer.nextToken());
558 assertEquals("a", clonedTokenizer.nextToken());
559 }
560
561 @Test
562 void testConstructor_charArray() {
563 StrTokenizer tok = new StrTokenizer("a b".toCharArray());
564 assertEquals("a", tok.next());
565 assertEquals("b", tok.next());
566 assertFalse(tok.hasNext());
567
568 tok = new StrTokenizer(ArrayUtils.EMPTY_CHAR_ARRAY);
569 assertFalse(tok.hasNext());
570
571 tok = new StrTokenizer((char[]) null);
572 assertFalse(tok.hasNext());
573 }
574
575 @Test
576 void testConstructor_charArray_char() {
577 StrTokenizer tok = new StrTokenizer("a b".toCharArray(), ' ');
578 assertEquals(1, tok.getDelimiterMatcher().isMatch(" ".toCharArray(), 0, 0, 1));
579 assertEquals("a", tok.next());
580 assertEquals("b", tok.next());
581 assertFalse(tok.hasNext());
582
583 tok = new StrTokenizer(ArrayUtils.EMPTY_CHAR_ARRAY, ' ');
584 assertFalse(tok.hasNext());
585
586 tok = new StrTokenizer((char[]) null, ' ');
587 assertFalse(tok.hasNext());
588 }
589
590 @Test
591 void testConstructor_charArray_char_char() {
592 StrTokenizer tok = new StrTokenizer("a b".toCharArray(), ' ', '"');
593 assertEquals(1, tok.getDelimiterMatcher().isMatch(" ".toCharArray(), 0, 0, 1));
594 assertEquals(1, tok.getQuoteMatcher().isMatch("\"".toCharArray(), 0, 0, 1));
595 assertEquals("a", tok.next());
596 assertEquals("b", tok.next());
597 assertFalse(tok.hasNext());
598
599 tok = new StrTokenizer(ArrayUtils.EMPTY_CHAR_ARRAY, ' ', '"');
600 assertFalse(tok.hasNext());
601
602 tok = new StrTokenizer((char[]) null, ' ', '"');
603 assertFalse(tok.hasNext());
604 }
605
606 @Test
607 void testConstructor_String() {
608 StrTokenizer tok = new StrTokenizer("a b");
609 assertEquals("a", tok.next());
610 assertEquals("b", tok.next());
611 assertFalse(tok.hasNext());
612
613 tok = new StrTokenizer("");
614 assertFalse(tok.hasNext());
615
616 tok = new StrTokenizer((String) null);
617 assertFalse(tok.hasNext());
618 }
619
620 @Test
621 void testConstructor_String_char() {
622 StrTokenizer tok = new StrTokenizer("a b", ' ');
623 assertEquals(1, tok.getDelimiterMatcher().isMatch(" ".toCharArray(), 0, 0, 1));
624 assertEquals("a", tok.next());
625 assertEquals("b", tok.next());
626 assertFalse(tok.hasNext());
627
628 tok = new StrTokenizer("", ' ');
629 assertFalse(tok.hasNext());
630
631 tok = new StrTokenizer((String) null, ' ');
632 assertFalse(tok.hasNext());
633 }
634
635 @Test
636 void testConstructor_String_char_char() {
637 StrTokenizer tok = new StrTokenizer("a b", ' ', '"');
638 assertEquals(1, tok.getDelimiterMatcher().isMatch(" ".toCharArray(), 0, 0, 1));
639 assertEquals(1, tok.getQuoteMatcher().isMatch("\"".toCharArray(), 0, 0, 1));
640 assertEquals("a", tok.next());
641 assertEquals("b", tok.next());
642 assertFalse(tok.hasNext());
643
644 tok = new StrTokenizer("", ' ', '"');
645 assertFalse(tok.hasNext());
646
647 tok = new StrTokenizer((String) null, ' ', '"');
648 assertFalse(tok.hasNext());
649 }
650
651 private void testCSV(final String data) {
652 testXSVAbc(StrTokenizer.getCSVInstance(data));
653 testXSVAbc(StrTokenizer.getCSVInstance(data.toCharArray()));
654 }
655
656 @Test
657 void testCSVEmpty() {
658 testEmpty(StrTokenizer.getCSVInstance());
659 testEmpty(StrTokenizer.getCSVInstance(""));
660 }
661
662 @Test
663 void testCSVSimple() {
664 testCSV(CSV_SIMPLE_FIXTURE);
665 }
666
667 @Test
668 void testCSVSimpleNeedsTrim() {
669 testCSV(" " + CSV_SIMPLE_FIXTURE);
670 testCSV(" \n\t " + CSV_SIMPLE_FIXTURE);
671 testCSV(" \n " + CSV_SIMPLE_FIXTURE + "\n\n\r");
672 }
673
674 @Test
675 void testDelimMatcher() {
676 final String input = "a/b\\c";
677 final StrMatcher delimMatcher = StrMatcher.charSetMatcher(new char[] {'/', '\\'});
678
679 final StrTokenizer tok = new StrTokenizer(input, delimMatcher);
680 assertEquals("a", tok.next());
681 assertEquals("b", tok.next());
682 assertEquals("c", tok.next());
683 assertFalse(tok.hasNext());
684 }
685
686 @Test
687 void testDelimMatcherQuoteMatcher() {
688 final String input = "`a`;`b`;`c`";
689 final StrMatcher delimMatcher = StrMatcher.charSetMatcher(new char[] {';'});
690 final StrMatcher quoteMatcher = StrMatcher.charSetMatcher(new char[] {'`'});
691
692 final StrTokenizer tok = new StrTokenizer(input, delimMatcher, quoteMatcher);
693 assertEquals("a", tok.next());
694 assertEquals("b", tok.next());
695 assertEquals("c", tok.next());
696 assertFalse(tok.hasNext());
697 }
698
699 @Test
700 void testDelimString() {
701 final String input = "a##b##c";
702 final StrTokenizer tok = new StrTokenizer(input, "##");
703
704 assertEquals("a", tok.next());
705 assertEquals("b", tok.next());
706 assertEquals("c", tok.next());
707 assertFalse(tok.hasNext());
708 }
709
710 void testEmpty(final StrTokenizer tokenizer) {
711 checkClone(tokenizer);
712 assertFalse(tokenizer.hasNext());
713 assertFalse(tokenizer.hasPrevious());
714 assertNull(tokenizer.nextToken());
715 assertEquals(0, tokenizer.size());
716 assertThrows(NoSuchElementException.class, tokenizer::next);
717 }
718
719 @Test
720 void testGetContent() {
721 final String input = "a b c \"d e\" f ";
722 StrTokenizer tok = new StrTokenizer(input);
723 assertEquals(input, tok.getContent());
724
725 tok = new StrTokenizer(input.toCharArray());
726 assertEquals(input, tok.getContent());
727
728 tok = new StrTokenizer();
729 assertNull(tok.getContent());
730 }
731
732 @Test
733 void testIteration() {
734 final StrTokenizer tkn = new StrTokenizer("a b c");
735 assertFalse(tkn.hasPrevious());
736 assertThrows(NoSuchElementException.class, tkn::previous);
737 assertTrue(tkn.hasNext());
738
739 assertEquals("a", tkn.next());
740 assertThrows(UnsupportedOperationException.class, tkn::remove);
741 assertThrows(UnsupportedOperationException.class, () -> tkn.set("x"));
742 assertThrows(UnsupportedOperationException.class, () -> tkn.add("y"));
743 assertTrue(tkn.hasPrevious());
744 assertTrue(tkn.hasNext());
745
746 assertEquals("b", tkn.next());
747 assertTrue(tkn.hasPrevious());
748 assertTrue(tkn.hasNext());
749
750 assertEquals("c", tkn.next());
751 assertTrue(tkn.hasPrevious());
752 assertFalse(tkn.hasNext());
753
754 assertThrows(NoSuchElementException.class, tkn::next);
755 assertTrue(tkn.hasPrevious());
756 assertFalse(tkn.hasNext());
757 }
758
759 @Test
760 void testListArray() {
761 final String input = "a b c";
762 final StrTokenizer tok = new StrTokenizer(input);
763 final String[] array = tok.getTokenArray();
764 final List<?> list = tok.getTokenList();
765
766 assertEquals(Arrays.asList(array), list);
767 assertEquals(3, list.size());
768 }
769
770 @Test
771 void testPreviousTokenAndSetEmptyTokenAsNull() {
772 final StrTokenizer strTokenizer = StrTokenizer.getTSVInstance(" \t\n\r\f");
773 strTokenizer.setEmptyTokenAsNull(true);
774
775 assertNull(strTokenizer.previousToken());
776 }
777
778 @Test
779 void testReset() {
780 final StrTokenizer tok = new StrTokenizer("a b c");
781 assertEquals("a", tok.next());
782 assertEquals("b", tok.next());
783 assertEquals("c", tok.next());
784 assertFalse(tok.hasNext());
785
786 tok.reset();
787 assertEquals("a", tok.next());
788 assertEquals("b", tok.next());
789 assertEquals("c", tok.next());
790 assertFalse(tok.hasNext());
791 }
792
793 @Test
794 void testReset_charArray() {
795 final StrTokenizer tok = new StrTokenizer("x x x");
796
797 final char[] array = {'a', 'b', 'c'};
798 tok.reset(array);
799 assertEquals("abc", tok.next());
800 assertFalse(tok.hasNext());
801
802 tok.reset((char[]) null);
803 assertFalse(tok.hasNext());
804 }
805
806 @Test
807 void testReset_String() {
808 final StrTokenizer tok = new StrTokenizer("x x x");
809 tok.reset("d e");
810 assertEquals("d", tok.next());
811 assertEquals("e", tok.next());
812 assertFalse(tok.hasNext());
813
814 tok.reset((String) null);
815 assertFalse(tok.hasNext());
816 }
817
818 @Test
819 void testStringTokenizerQuoteMatcher() {
820 final char[] chars = {'\'', 'a', 'c', '\'', 'd'};
821 final StrTokenizer tokens = new StrTokenizer(chars, StrMatcher.commaMatcher(), StrMatcher.quoteMatcher());
822 assertEquals("acd", tokens.next());
823 }
824
825 @Test
826 void testStringTokenizerStringMatcher() {
827 final char[] chars = {'a', 'b', 'c', 'd'};
828 final StrTokenizer tokens = new StrTokenizer(chars, "bc");
829 assertEquals("a", tokens.next());
830 assertEquals("d", tokens.next());
831 }
832
833 @Test
834 void testStringTokenizerStrMatcher() {
835 final char[] chars = {'a', ',', 'c'};
836 final StrTokenizer tokens = new StrTokenizer(chars, StrMatcher.commaMatcher());
837 assertEquals("a", tokens.next());
838 assertEquals("c", tokens.next());
839 }
840
841 @Test
842 void testTokenizeSubclassInputChange() {
843 final StrTokenizer tkn = new StrTokenizer("a b c d e") {
844
845 @Override
846 protected List<String> tokenize(final char[] chars, final int offset, final int count) {
847 return super.tokenize("w x y z".toCharArray(), 2, 5);
848 }
849 };
850 assertEquals("x", tkn.next());
851 assertEquals("y", tkn.next());
852 }
853
854 @Test
855 void testTokenizeSubclassOutputChange() {
856 final StrTokenizer tkn = new StrTokenizer("a b c") {
857
858 @Override
859 protected List<String> tokenize(final char[] chars, final int offset, final int count) {
860 final List<String> list = super.tokenize(chars, offset, count);
861 Collections.reverse(list);
862 return list;
863 }
864 };
865 assertEquals("c", tkn.next());
866 assertEquals("b", tkn.next());
867 assertEquals("a", tkn.next());
868 }
869
870 @Test
871 void testToString() {
872 final StrTokenizer tkn = new StrTokenizer("a b c d e");
873 assertEquals("StrTokenizer[not tokenized yet]", tkn.toString());
874 tkn.next();
875 assertEquals("StrTokenizer[a, b, c, d, e]", tkn.toString());
876 }
877
878 @Test
879 void testTSV() {
880 testXSVAbc(StrTokenizer.getTSVInstance(TSV_SIMPLE_FIXTURE));
881 testXSVAbc(StrTokenizer.getTSVInstance(TSV_SIMPLE_FIXTURE.toCharArray()));
882 }
883
884 @Test
885 void testTSVEmpty() {
886 testEmpty(StrTokenizer.getTSVInstance());
887 testEmpty(StrTokenizer.getTSVInstance(""));
888 }
889
890 void testXSVAbc(final StrTokenizer tokenizer) {
891 checkClone(tokenizer);
892 assertEquals(-1, tokenizer.previousIndex());
893 assertEquals(0, tokenizer.nextIndex());
894 assertNull(tokenizer.previousToken());
895 assertEquals("A", tokenizer.nextToken());
896 assertEquals(1, tokenizer.nextIndex());
897 assertEquals("b", tokenizer.nextToken());
898 assertEquals(2, tokenizer.nextIndex());
899 assertEquals("c", tokenizer.nextToken());
900 assertEquals(3, tokenizer.nextIndex());
901 assertNull(tokenizer.nextToken());
902 assertEquals(3, tokenizer.nextIndex());
903 assertEquals("c", tokenizer.previousToken());
904 assertEquals(2, tokenizer.nextIndex());
905 assertEquals("b", tokenizer.previousToken());
906 assertEquals(1, tokenizer.nextIndex());
907 assertEquals("A", tokenizer.previousToken());
908 assertEquals(0, tokenizer.nextIndex());
909 assertNull(tokenizer.previousToken());
910 assertEquals(0, tokenizer.nextIndex());
911 assertEquals(-1, tokenizer.previousIndex());
912 assertEquals(3, tokenizer.size());
913 }
914 }