1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.commons.text;
19
20 import static org.junit.jupiter.api.Assertions.assertEquals;
21 import static org.junit.jupiter.api.Assertions.assertFalse;
22 import static org.junit.jupiter.api.Assertions.assertNotSame;
23 import static org.junit.jupiter.api.Assertions.assertNull;
24 import static org.junit.jupiter.api.Assertions.assertThrows;
25 import static org.junit.jupiter.api.Assertions.assertTrue;
26
27 import java.util.Arrays;
28 import java.util.Collections;
29 import java.util.List;
30 import java.util.NoSuchElementException;
31
32 import org.apache.commons.lang3.ArrayUtils;
33 import org.junit.jupiter.api.Test;
34
35
36
37
38
39
40 @Deprecated
41 class StrTokenizerTest {
42
43 private static final String CSV_SIMPLE_FIXTURE = "A,b,c";
44
45 private static final String TSV_SIMPLE_FIXTURE = "A\tb\tc";
46
47 private void checkClone(final StrTokenizer tokenizer) {
48 assertNotSame(StrTokenizer.getCSVInstance(), tokenizer);
49 assertNotSame(StrTokenizer.getTSVInstance(), tokenizer);
50 }
51
52 @Test
53 void test1() {
54
55 final String input = "a;b;c;\"d;\"\"e\";f; ; ; ";
56 final StrTokenizer tok = new StrTokenizer(input);
57 tok.setDelimiterChar(';');
58 tok.setQuoteChar('"');
59 tok.setIgnoredMatcher(StrMatcher.trimMatcher());
60 tok.setIgnoreEmptyTokens(false);
61 final String[] tokens = tok.getTokenArray();
62
63 final String[] expected = {"a", "b", "c", "d;\"e", "f", "", "", ""};
64
65 assertEquals(expected.length, tokens.length, Arrays.toString(tokens));
66 for (int i = 0; i < expected.length; i++) {
67 assertEquals(expected[i], tokens[i],
68 "token[" + i + "] was '" + tokens[i] + "' but was expected to be '" + expected[i] + "'");
69 }
70
71 }
72
73 @Test
74 void test2() {
75
76 final String input = "a;b;c ;\"d;\"\"e\";f; ; ;";
77 final StrTokenizer tok = new StrTokenizer(input);
78 tok.setDelimiterChar(';');
79 tok.setQuoteChar('"');
80 tok.setIgnoredMatcher(StrMatcher.noneMatcher());
81 tok.setIgnoreEmptyTokens(false);
82 final String[] tokens = tok.getTokenArray();
83
84 final String[] expected = {"a", "b", "c ", "d;\"e", "f", " ", " ", ""};
85
86 assertEquals(expected.length, tokens.length, Arrays.toString(tokens));
87 for (int i = 0; i < expected.length; i++) {
88 assertEquals(expected[i], tokens[i],
89 "token[" + i + "] was '" + tokens[i] + "' but was expected to be '" + expected[i] + "'");
90 }
91
92 }
93
94 @Test
95 void test3() {
96
97 final String input = "a;b; c;\"d;\"\"e\";f; ; ;";
98 final StrTokenizer tok = new StrTokenizer(input);
99 tok.setDelimiterChar(';');
100 tok.setQuoteChar('"');
101 tok.setIgnoredMatcher(StrMatcher.noneMatcher());
102 tok.setIgnoreEmptyTokens(false);
103 final String[] tokens = tok.getTokenArray();
104
105 final String[] expected = {"a", "b", " c", "d;\"e", "f", " ", " ", ""};
106
107 assertEquals(expected.length, tokens.length, Arrays.toString(tokens));
108 for (int i = 0; i < expected.length; i++) {
109 assertEquals(expected[i], tokens[i],
110 "token[" + i + "] was '" + tokens[i] + "' but was expected to be '" + expected[i] + "'");
111 }
112
113 }
114
115 @Test
116 void test4() {
117
118 final String input = "a;b; c;\"d;\"\"e\";f; ; ;";
119 final StrTokenizer tok = new StrTokenizer(input);
120 tok.setDelimiterChar(';');
121 tok.setQuoteChar('"');
122 tok.setIgnoredMatcher(StrMatcher.trimMatcher());
123 tok.setIgnoreEmptyTokens(true);
124 final String[] tokens = tok.getTokenArray();
125
126 final String[] expected = {"a", "b", "c", "d;\"e", "f"};
127
128 assertEquals(expected.length, tokens.length, Arrays.toString(tokens));
129 for (int i = 0; i < expected.length; i++) {
130 assertEquals(expected[i], tokens[i],
131 "token[" + i + "] was '" + tokens[i] + "' but was expected to be '" + expected[i] + "'");
132 }
133
134 }
135
136 @Test
137 void test5() {
138
139 final String input = "a;b; c;\"d;\"\"e\";f; ; ;";
140 final StrTokenizer tok = new StrTokenizer(input);
141 tok.setDelimiterChar(';');
142 tok.setQuoteChar('"');
143 tok.setIgnoredMatcher(StrMatcher.trimMatcher());
144 tok.setIgnoreEmptyTokens(false);
145 tok.setEmptyTokenAsNull(true);
146 final String[] tokens = tok.getTokenArray();
147
148 final String[] expected = {"a", "b", "c", "d;\"e", "f", null, null, null};
149
150 assertEquals(expected.length, tokens.length, Arrays.toString(tokens));
151 for (int i = 0; i < expected.length; i++) {
152 assertEquals(expected[i], tokens[i],
153 "token[" + i + "] was '" + tokens[i] + "' but was expected to be '" + expected[i] + "'");
154 }
155
156 }
157
158 @Test
159 void test6() {
160
161 final String input = "a;b; c;\"d;\"\"e\";f; ; ;";
162 final StrTokenizer tok = new StrTokenizer(input);
163 tok.setDelimiterChar(';');
164 tok.setQuoteChar('"');
165 tok.setIgnoredMatcher(StrMatcher.trimMatcher());
166 tok.setIgnoreEmptyTokens(false);
167
168 final String[] tokens = tok.getTokenArray();
169
170 final String[] expected = {"a", "b", " c", "d;\"e", "f", null, null, null};
171
172 int nextCount = 0;
173 while (tok.hasNext()) {
174 tok.next();
175 nextCount++;
176 }
177
178 int prevCount = 0;
179 while (tok.hasPrevious()) {
180 tok.previous();
181 prevCount++;
182 }
183
184 assertEquals(expected.length, tokens.length, Arrays.toString(tokens));
185 assertEquals(nextCount, expected.length, "could not cycle through entire token list using the 'hasNext' and 'next' methods");
186 assertEquals(prevCount, expected.length, "could not cycle through entire token list using the 'hasPrevious' and 'previous' methods");
187 }
188
189 @Test
190 void test7() {
191
192 final String input = "a b c \"d e\" f ";
193 final StrTokenizer tok = new StrTokenizer(input);
194 tok.setDelimiterMatcher(StrMatcher.spaceMatcher());
195 tok.setQuoteMatcher(StrMatcher.doubleQuoteMatcher());
196 tok.setIgnoredMatcher(StrMatcher.noneMatcher());
197 tok.setIgnoreEmptyTokens(false);
198 final String[] tokens = tok.getTokenArray();
199
200 final String[] expected = {"a", "", "", "b", "c", "d e", "f", ""};
201
202 assertEquals(expected.length, tokens.length, Arrays.toString(tokens));
203 for (int i = 0; i < expected.length; i++) {
204 assertEquals(expected[i], tokens[i],
205 "token[" + i + "] was '" + tokens[i] + "' but was expected to be '" + expected[i] + "'");
206 }
207
208 }
209
210 @Test
211 void test8() {
212
213 final String input = "a b c \"d e\" f ";
214 final StrTokenizer tok = new StrTokenizer(input);
215 tok.setDelimiterMatcher(StrMatcher.spaceMatcher());
216 tok.setQuoteMatcher(StrMatcher.doubleQuoteMatcher());
217 tok.setIgnoredMatcher(StrMatcher.noneMatcher());
218 tok.setIgnoreEmptyTokens(true);
219 final String[] tokens = tok.getTokenArray();
220
221 final String[] expected = {"a", "b", "c", "d e", "f"};
222
223 assertEquals(expected.length, tokens.length, Arrays.toString(tokens));
224 for (int i = 0; i < expected.length; i++) {
225 assertEquals(expected[i], tokens[i],
226 "token[" + i + "] was '" + tokens[i] + "' but was expected to be '" + expected[i] + "'");
227 }
228
229 }
230
231 @Test
232 void testBasic1() {
233 final String input = "a b c";
234 final StrTokenizer tok = new StrTokenizer(input);
235 assertEquals("a", tok.next());
236 assertEquals("b", tok.next());
237 assertEquals("c", tok.next());
238 assertFalse(tok.hasNext());
239 }
240
241 @Test
242 void testBasic2() {
243 final String input = "a \nb\fc";
244 final StrTokenizer tok = new StrTokenizer(input);
245 assertEquals("a", tok.next());
246 assertEquals("b", tok.next());
247 assertEquals("c", tok.next());
248 assertFalse(tok.hasNext());
249 }
250
251 @Test
252 void testBasic3() {
253 final String input = "a \nb\u0001\fc";
254 final StrTokenizer tok = new StrTokenizer(input);
255 assertEquals("a", tok.next());
256 assertEquals("b\u0001", tok.next());
257 assertEquals("c", tok.next());
258 assertFalse(tok.hasNext());
259 }
260
261 @Test
262 void testBasic4() {
263 final String input = "a \"b\" c";
264 final StrTokenizer tok = new StrTokenizer(input);
265 assertEquals("a", tok.next());
266 assertEquals("\"b\"", tok.next());
267 assertEquals("c", tok.next());
268 assertFalse(tok.hasNext());
269 }
270
271 @Test
272 void testBasic5() {
273 final String input = "a:b':c";
274 final StrTokenizer tok = new StrTokenizer(input, ':', '\'');
275 assertEquals("a", tok.next());
276 assertEquals("b'", tok.next());
277 assertEquals("c", tok.next());
278 assertFalse(tok.hasNext());
279 }
280
281 @Test
282 void testBasicDelim1() {
283 final String input = "a:b:c";
284 final StrTokenizer tok = new StrTokenizer(input, ':');
285 assertEquals("a", tok.next());
286 assertEquals("b", tok.next());
287 assertEquals("c", tok.next());
288 assertFalse(tok.hasNext());
289 }
290
291 @Test
292 void testBasicDelim2() {
293 final String input = "a:b:c";
294 final StrTokenizer tok = new StrTokenizer(input, ',');
295 assertEquals("a:b:c", tok.next());
296 assertFalse(tok.hasNext());
297 }
298
299 @Test
300 void testBasicEmpty1() {
301 final String input = "a b c";
302 final StrTokenizer tok = new StrTokenizer(input);
303 tok.setIgnoreEmptyTokens(false);
304 assertEquals("a", tok.next());
305 assertEquals("", tok.next());
306 assertEquals("b", tok.next());
307 assertEquals("c", tok.next());
308 assertFalse(tok.hasNext());
309 }
310
311 @Test
312 void testBasicEmpty2() {
313 final String input = "a b c";
314 final StrTokenizer tok = new StrTokenizer(input);
315 tok.setIgnoreEmptyTokens(false);
316 tok.setEmptyTokenAsNull(true);
317 assertEquals("a", tok.next());
318 assertNull(tok.next());
319 assertEquals("b", tok.next());
320 assertEquals("c", tok.next());
321 assertFalse(tok.hasNext());
322 }
323
324 @Test
325 void testBasicIgnoreTrimmed1() {
326 final String input = "a: bIGNOREc : ";
327 final StrTokenizer tok = new StrTokenizer(input, ':');
328 tok.setIgnoredMatcher(StrMatcher.stringMatcher("IGNORE"));
329 tok.setTrimmerMatcher(StrMatcher.trimMatcher());
330 tok.setIgnoreEmptyTokens(false);
331 tok.setEmptyTokenAsNull(true);
332 assertEquals("a", tok.next());
333 assertEquals("bc", tok.next());
334 assertNull(tok.next());
335 assertFalse(tok.hasNext());
336 }
337
338 @Test
339 void testBasicIgnoreTrimmed2() {
340 final String input = "IGNOREaIGNORE: IGNORE bIGNOREc IGNORE : IGNORE ";
341 final StrTokenizer tok = new StrTokenizer(input, ':');
342 tok.setIgnoredMatcher(StrMatcher.stringMatcher("IGNORE"));
343 tok.setTrimmerMatcher(StrMatcher.trimMatcher());
344 tok.setIgnoreEmptyTokens(false);
345 tok.setEmptyTokenAsNull(true);
346 assertEquals("a", tok.next());
347 assertEquals("bc", tok.next());
348 assertNull(tok.next());
349 assertFalse(tok.hasNext());
350 }
351
352 @Test
353 void testBasicIgnoreTrimmed3() {
354 final String input = "IGNOREaIGNORE: IGNORE bIGNOREc IGNORE : IGNORE ";
355 final StrTokenizer tok = new StrTokenizer(input, ':');
356 tok.setIgnoredMatcher(StrMatcher.stringMatcher("IGNORE"));
357 tok.setIgnoreEmptyTokens(false);
358 tok.setEmptyTokenAsNull(true);
359 assertEquals("a", tok.next());
360 assertEquals(" bc ", tok.next());
361 assertEquals(" ", tok.next());
362 assertFalse(tok.hasNext());
363 }
364
365 @Test
366 void testBasicIgnoreTrimmed4() {
367 final String input = "IGNOREaIGNORE: IGNORE 'bIGNOREc'IGNORE'd' IGNORE : IGNORE ";
368 final StrTokenizer tok = new StrTokenizer(input, ':', '\'');
369 tok.setIgnoredMatcher(StrMatcher.stringMatcher("IGNORE"));
370 tok.setTrimmerMatcher(StrMatcher.trimMatcher());
371 tok.setIgnoreEmptyTokens(false);
372 tok.setEmptyTokenAsNull(true);
373 assertEquals("a", tok.next());
374 assertEquals("bIGNOREcd", tok.next());
375 assertNull(tok.next());
376 assertFalse(tok.hasNext());
377 }
378
379 @Test
380 void testBasicQuoted1() {
381 final String input = "a 'b' c";
382 final StrTokenizer tok = new StrTokenizer(input, ' ', '\'');
383 assertEquals("a", tok.next());
384 assertEquals("b", tok.next());
385 assertEquals("c", tok.next());
386 assertFalse(tok.hasNext());
387 }
388
389 @Test
390 void testBasicQuoted2() {
391 final String input = "a:'b':";
392 final StrTokenizer tok = new StrTokenizer(input, ':', '\'');
393 tok.setIgnoreEmptyTokens(false);
394 tok.setEmptyTokenAsNull(true);
395 assertEquals("a", tok.next());
396 assertEquals("b", tok.next());
397 assertNull(tok.next());
398 assertFalse(tok.hasNext());
399 }
400
401 @Test
402 void testBasicQuoted3() {
403 final String input = "a:'b''c'";
404 final StrTokenizer tok = new StrTokenizer(input, ':', '\'');
405 tok.setIgnoreEmptyTokens(false);
406 tok.setEmptyTokenAsNull(true);
407 assertEquals("a", tok.next());
408 assertEquals("b'c", tok.next());
409 assertFalse(tok.hasNext());
410 }
411
412 @Test
413 void testBasicQuoted4() {
414 final String input = "a: 'b' 'c' :d";
415 final StrTokenizer tok = new StrTokenizer(input, ':', '\'');
416 tok.setTrimmerMatcher(StrMatcher.trimMatcher());
417 tok.setIgnoreEmptyTokens(false);
418 tok.setEmptyTokenAsNull(true);
419 assertEquals("a", tok.next());
420 assertEquals("b c", tok.next());
421 assertEquals("d", tok.next());
422 assertFalse(tok.hasNext());
423 }
424
425 @Test
426 void testBasicQuoted5() {
427 final String input = "a: 'b'x'c' :d";
428 final StrTokenizer tok = new StrTokenizer(input, ':', '\'');
429 tok.setTrimmerMatcher(StrMatcher.trimMatcher());
430 tok.setIgnoreEmptyTokens(false);
431 tok.setEmptyTokenAsNull(true);
432 assertEquals("a", tok.next());
433 assertEquals("bxc", tok.next());
434 assertEquals("d", tok.next());
435 assertFalse(tok.hasNext());
436 }
437
438 @Test
439 void testBasicQuoted6() {
440 final String input = "a:'b'\"c':d";
441 final StrTokenizer tok = new StrTokenizer(input, ':');
442 tok.setQuoteMatcher(StrMatcher.quoteMatcher());
443 assertEquals("a", tok.next());
444 assertEquals("b\"c:d", tok.next());
445 assertFalse(tok.hasNext());
446 }
447
448 @Test
449 void testBasicQuoted7() {
450 final String input = "a:\"There's a reason here\":b";
451 final StrTokenizer tok = new StrTokenizer(input, ':');
452 tok.setQuoteMatcher(StrMatcher.quoteMatcher());
453 assertEquals("a", tok.next());
454 assertEquals("There's a reason here", tok.next());
455 assertEquals("b", tok.next());
456 assertFalse(tok.hasNext());
457 }
458
459 @Test
460 void testBasicQuotedTrimmed1() {
461 final String input = "a: 'b' :";
462 final StrTokenizer tok = new StrTokenizer(input, ':', '\'');
463 tok.setTrimmerMatcher(StrMatcher.trimMatcher());
464 tok.setIgnoreEmptyTokens(false);
465 tok.setEmptyTokenAsNull(true);
466 assertEquals("a", tok.next());
467 assertEquals("b", tok.next());
468 assertNull(tok.next());
469 assertFalse(tok.hasNext());
470 }
471
472 @Test
473 void testBasicTrimmed1() {
474 final String input = "a: b : ";
475 final StrTokenizer tok = new StrTokenizer(input, ':');
476 tok.setTrimmerMatcher(StrMatcher.trimMatcher());
477 tok.setIgnoreEmptyTokens(false);
478 tok.setEmptyTokenAsNull(true);
479 assertEquals("a", tok.next());
480 assertEquals("b", tok.next());
481 assertNull(tok.next());
482 assertFalse(tok.hasNext());
483 }
484
485 @Test
486 void testBasicTrimmed2() {
487 final String input = "a: b :";
488 final StrTokenizer tok = new StrTokenizer(input, ':');
489 tok.setTrimmerMatcher(StrMatcher.stringMatcher(" "));
490 tok.setIgnoreEmptyTokens(false);
491 tok.setEmptyTokenAsNull(true);
492 assertEquals("a", tok.next());
493 assertEquals("b", tok.next());
494 assertNull(tok.next());
495 assertFalse(tok.hasNext());
496 }
497
498 @Test
499 void testChaining() {
500 final StrTokenizer tok = new StrTokenizer();
501 assertEquals(tok, tok.reset());
502 assertEquals(tok, tok.reset(""));
503 assertEquals(tok, tok.reset(ArrayUtils.EMPTY_CHAR_ARRAY));
504 assertEquals(tok, tok.setDelimiterChar(' '));
505 assertEquals(tok, tok.setDelimiterString(" "));
506 assertEquals(tok, tok.setDelimiterMatcher(null));
507 assertEquals(tok, tok.setQuoteChar(' '));
508 assertEquals(tok, tok.setQuoteMatcher(null));
509 assertEquals(tok, tok.setIgnoredChar(' '));
510 assertEquals(tok, tok.setIgnoredMatcher(null));
511 assertEquals(tok, tok.setTrimmerMatcher(null));
512 assertEquals(tok, tok.setEmptyTokenAsNull(false));
513 assertEquals(tok, tok.setIgnoreEmptyTokens(false));
514 }
515
516
517
518
519
520 @Test
521 void testCloneNotSupportedException() {
522 final Object notCloned = new StrTokenizer() {
523
524 @Override
525 Object cloneReset() throws CloneNotSupportedException {
526 throw new CloneNotSupportedException("test");
527 }
528 }.clone();
529 assertNull(notCloned);
530 }
531
532 @Test
533 void testCloneNull() {
534 final StrTokenizer tokenizer = new StrTokenizer((char[]) null);
535
536 assertNull(tokenizer.nextToken());
537 tokenizer.reset();
538 assertNull(tokenizer.nextToken());
539
540 final StrTokenizer clonedTokenizer = (StrTokenizer) tokenizer.clone();
541 tokenizer.reset();
542 assertNull(tokenizer.nextToken());
543 assertNull(clonedTokenizer.nextToken());
544 }
545
546 @Test
547 void testCloneReset() {
548 final char[] input = {'a'};
549 final StrTokenizer tokenizer = new StrTokenizer(input);
550
551 assertEquals("a", tokenizer.nextToken());
552 tokenizer.reset(input);
553 assertEquals("a", tokenizer.nextToken());
554
555 final StrTokenizer clonedTokenizer = (StrTokenizer) tokenizer.clone();
556 input[0] = 'b';
557 tokenizer.reset(input);
558 assertEquals("b", tokenizer.nextToken());
559 assertEquals("a", clonedTokenizer.nextToken());
560 }
561
562 @Test
563 void testConstructor_charArray() {
564 StrTokenizer tok = new StrTokenizer("a b".toCharArray());
565 assertEquals("a", tok.next());
566 assertEquals("b", tok.next());
567 assertFalse(tok.hasNext());
568
569 tok = new StrTokenizer(ArrayUtils.EMPTY_CHAR_ARRAY);
570 assertFalse(tok.hasNext());
571
572 tok = new StrTokenizer((char[]) null);
573 assertFalse(tok.hasNext());
574 }
575
576 @Test
577 void testConstructor_charArray_char() {
578 StrTokenizer tok = new StrTokenizer("a b".toCharArray(), ' ');
579 assertEquals(1, tok.getDelimiterMatcher().isMatch(" ".toCharArray(), 0, 0, 1));
580 assertEquals("a", tok.next());
581 assertEquals("b", tok.next());
582 assertFalse(tok.hasNext());
583
584 tok = new StrTokenizer(ArrayUtils.EMPTY_CHAR_ARRAY, ' ');
585 assertFalse(tok.hasNext());
586
587 tok = new StrTokenizer((char[]) null, ' ');
588 assertFalse(tok.hasNext());
589 }
590
591 @Test
592 void testConstructor_charArray_char_char() {
593 StrTokenizer tok = new StrTokenizer("a b".toCharArray(), ' ', '"');
594 assertEquals(1, tok.getDelimiterMatcher().isMatch(" ".toCharArray(), 0, 0, 1));
595 assertEquals(1, tok.getQuoteMatcher().isMatch("\"".toCharArray(), 0, 0, 1));
596 assertEquals("a", tok.next());
597 assertEquals("b", tok.next());
598 assertFalse(tok.hasNext());
599
600 tok = new StrTokenizer(ArrayUtils.EMPTY_CHAR_ARRAY, ' ', '"');
601 assertFalse(tok.hasNext());
602
603 tok = new StrTokenizer((char[]) null, ' ', '"');
604 assertFalse(tok.hasNext());
605 }
606
607 @Test
608 void testConstructor_String() {
609 StrTokenizer tok = new StrTokenizer("a b");
610 assertEquals("a", tok.next());
611 assertEquals("b", tok.next());
612 assertFalse(tok.hasNext());
613
614 tok = new StrTokenizer("");
615 assertFalse(tok.hasNext());
616
617 tok = new StrTokenizer((String) null);
618 assertFalse(tok.hasNext());
619 }
620
621 @Test
622 void testConstructor_String_char() {
623 StrTokenizer tok = new StrTokenizer("a b", ' ');
624 assertEquals(1, tok.getDelimiterMatcher().isMatch(" ".toCharArray(), 0, 0, 1));
625 assertEquals("a", tok.next());
626 assertEquals("b", tok.next());
627 assertFalse(tok.hasNext());
628
629 tok = new StrTokenizer("", ' ');
630 assertFalse(tok.hasNext());
631
632 tok = new StrTokenizer((String) null, ' ');
633 assertFalse(tok.hasNext());
634 }
635
636 @Test
637 void testConstructor_String_char_char() {
638 StrTokenizer tok = new StrTokenizer("a b", ' ', '"');
639 assertEquals(1, tok.getDelimiterMatcher().isMatch(" ".toCharArray(), 0, 0, 1));
640 assertEquals(1, tok.getQuoteMatcher().isMatch("\"".toCharArray(), 0, 0, 1));
641 assertEquals("a", tok.next());
642 assertEquals("b", tok.next());
643 assertFalse(tok.hasNext());
644
645 tok = new StrTokenizer("", ' ', '"');
646 assertFalse(tok.hasNext());
647
648 tok = new StrTokenizer((String) null, ' ', '"');
649 assertFalse(tok.hasNext());
650 }
651
652 private void testCSV(final String data) {
653 testXSVAbc(StrTokenizer.getCSVInstance(data));
654 testXSVAbc(StrTokenizer.getCSVInstance(data.toCharArray()));
655 }
656
657 @Test
658 void testCSVEmpty() {
659 testEmpty(StrTokenizer.getCSVInstance());
660 testEmpty(StrTokenizer.getCSVInstance(""));
661 }
662
663 @Test
664 void testCSVSimple() {
665 testCSV(CSV_SIMPLE_FIXTURE);
666 }
667
668 @Test
669 void testCSVSimpleNeedsTrim() {
670 testCSV(" " + CSV_SIMPLE_FIXTURE);
671 testCSV(" \n\t " + CSV_SIMPLE_FIXTURE);
672 testCSV(" \n " + CSV_SIMPLE_FIXTURE + "\n\n\r");
673 }
674
675 @Test
676 void testDelimMatcher() {
677 final String input = "a/b\\c";
678 final StrMatcher delimMatcher = StrMatcher.charSetMatcher(new char[] {'/', '\\'});
679
680 final StrTokenizer tok = new StrTokenizer(input, delimMatcher);
681 assertEquals("a", tok.next());
682 assertEquals("b", tok.next());
683 assertEquals("c", tok.next());
684 assertFalse(tok.hasNext());
685 }
686
687 @Test
688 void testDelimMatcherQuoteMatcher() {
689 final String input = "`a`;`b`;`c`";
690 final StrMatcher delimMatcher = StrMatcher.charSetMatcher(new char[] {';'});
691 final StrMatcher quoteMatcher = StrMatcher.charSetMatcher(new char[] {'`'});
692
693 final StrTokenizer tok = new StrTokenizer(input, delimMatcher, quoteMatcher);
694 assertEquals("a", tok.next());
695 assertEquals("b", tok.next());
696 assertEquals("c", tok.next());
697 assertFalse(tok.hasNext());
698 }
699
700 @Test
701 void testDelimString() {
702 final String input = "a##b##c";
703 final StrTokenizer tok = new StrTokenizer(input, "##");
704
705 assertEquals("a", tok.next());
706 assertEquals("b", tok.next());
707 assertEquals("c", tok.next());
708 assertFalse(tok.hasNext());
709 }
710
711 void testEmpty(final StrTokenizer tokenizer) {
712 checkClone(tokenizer);
713 assertFalse(tokenizer.hasNext());
714 assertFalse(tokenizer.hasPrevious());
715 assertNull(tokenizer.nextToken());
716 assertEquals(0, tokenizer.size());
717 assertThrows(NoSuchElementException.class, tokenizer::next);
718 }
719
720 @Test
721 void testGetContent() {
722 final String input = "a b c \"d e\" f ";
723 StrTokenizer tok = new StrTokenizer(input);
724 assertEquals(input, tok.getContent());
725
726 tok = new StrTokenizer(input.toCharArray());
727 assertEquals(input, tok.getContent());
728
729 tok = new StrTokenizer();
730 assertNull(tok.getContent());
731 }
732
733 @Test
734 void testIteration() {
735 final StrTokenizer tkn = new StrTokenizer("a b c");
736 assertFalse(tkn.hasPrevious());
737 assertThrows(NoSuchElementException.class, tkn::previous);
738 assertTrue(tkn.hasNext());
739
740 assertEquals("a", tkn.next());
741 assertThrows(UnsupportedOperationException.class, tkn::remove);
742 assertThrows(UnsupportedOperationException.class, () -> tkn.set("x"));
743 assertThrows(UnsupportedOperationException.class, () -> tkn.add("y"));
744 assertTrue(tkn.hasPrevious());
745 assertTrue(tkn.hasNext());
746
747 assertEquals("b", tkn.next());
748 assertTrue(tkn.hasPrevious());
749 assertTrue(tkn.hasNext());
750
751 assertEquals("c", tkn.next());
752 assertTrue(tkn.hasPrevious());
753 assertFalse(tkn.hasNext());
754
755 assertThrows(NoSuchElementException.class, tkn::next);
756 assertTrue(tkn.hasPrevious());
757 assertFalse(tkn.hasNext());
758 }
759
760 @Test
761 void testListArray() {
762 final String input = "a b c";
763 final StrTokenizer tok = new StrTokenizer(input);
764 final String[] array = tok.getTokenArray();
765 final List<?> list = tok.getTokenList();
766
767 assertEquals(Arrays.asList(array), list);
768 assertEquals(3, list.size());
769 }
770
771 @Test
772 void testPreviousTokenAndSetEmptyTokenAsNull() {
773 final StrTokenizer strTokenizer = StrTokenizer.getTSVInstance(" \t\n\r\f");
774 strTokenizer.setEmptyTokenAsNull(true);
775
776 assertNull(strTokenizer.previousToken());
777 }
778
779 @Test
780 void testReset() {
781 final StrTokenizer tok = new StrTokenizer("a b c");
782 assertEquals("a", tok.next());
783 assertEquals("b", tok.next());
784 assertEquals("c", tok.next());
785 assertFalse(tok.hasNext());
786
787 tok.reset();
788 assertEquals("a", tok.next());
789 assertEquals("b", tok.next());
790 assertEquals("c", tok.next());
791 assertFalse(tok.hasNext());
792 }
793
794 @Test
795 void testReset_charArray() {
796 final StrTokenizer tok = new StrTokenizer("x x x");
797
798 final char[] array = {'a', 'b', 'c'};
799 tok.reset(array);
800 assertEquals("abc", tok.next());
801 assertFalse(tok.hasNext());
802
803 tok.reset((char[]) null);
804 assertFalse(tok.hasNext());
805 }
806
807 @Test
808 void testReset_String() {
809 final StrTokenizer tok = new StrTokenizer("x x x");
810 tok.reset("d e");
811 assertEquals("d", tok.next());
812 assertEquals("e", tok.next());
813 assertFalse(tok.hasNext());
814
815 tok.reset((String) null);
816 assertFalse(tok.hasNext());
817 }
818
819 @Test
820 void testStringTokenizerQuoteMatcher() {
821 final char[] chars = {'\'', 'a', 'c', '\'', 'd'};
822 final StrTokenizer tokens = new StrTokenizer(chars, StrMatcher.commaMatcher(), StrMatcher.quoteMatcher());
823 assertEquals("acd", tokens.next());
824 }
825
826 @Test
827 void testStringTokenizerStringMatcher() {
828 final char[] chars = {'a', 'b', 'c', 'd'};
829 final StrTokenizer tokens = new StrTokenizer(chars, "bc");
830 assertEquals("a", tokens.next());
831 assertEquals("d", tokens.next());
832 }
833
834 @Test
835 void testStringTokenizerStrMatcher() {
836 final char[] chars = {'a', ',', 'c'};
837 final StrTokenizer tokens = new StrTokenizer(chars, StrMatcher.commaMatcher());
838 assertEquals("a", tokens.next());
839 assertEquals("c", tokens.next());
840 }
841
842 @Test
843 void testTokenizeSubclassInputChange() {
844 final StrTokenizer tkn = new StrTokenizer("a b c d e") {
845
846 @Override
847 protected List<String> tokenize(final char[] chars, final int offset, final int count) {
848 return super.tokenize("w x y z".toCharArray(), 2, 5);
849 }
850 };
851 assertEquals("x", tkn.next());
852 assertEquals("y", tkn.next());
853 }
854
855 @Test
856 void testTokenizeSubclassOutputChange() {
857 final StrTokenizer tkn = new StrTokenizer("a b c") {
858
859 @Override
860 protected List<String> tokenize(final char[] chars, final int offset, final int count) {
861 final List<String> list = super.tokenize(chars, offset, count);
862 Collections.reverse(list);
863 return list;
864 }
865 };
866 assertEquals("c", tkn.next());
867 assertEquals("b", tkn.next());
868 assertEquals("a", tkn.next());
869 }
870
871 @Test
872 void testToString() {
873 final StrTokenizer tkn = new StrTokenizer("a b c d e");
874 assertEquals("StrTokenizer[not tokenized yet]", tkn.toString());
875 tkn.next();
876 assertEquals("StrTokenizer[a, b, c, d, e]", tkn.toString());
877 }
878
879 @Test
880 void testTSV() {
881 testXSVAbc(StrTokenizer.getTSVInstance(TSV_SIMPLE_FIXTURE));
882 testXSVAbc(StrTokenizer.getTSVInstance(TSV_SIMPLE_FIXTURE.toCharArray()));
883 }
884
885 @Test
886 void testTSVEmpty() {
887 testEmpty(StrTokenizer.getTSVInstance());
888 testEmpty(StrTokenizer.getTSVInstance(""));
889 }
890
891 void testXSVAbc(final StrTokenizer tokenizer) {
892 checkClone(tokenizer);
893 assertEquals(-1, tokenizer.previousIndex());
894 assertEquals(0, tokenizer.nextIndex());
895 assertNull(tokenizer.previousToken());
896 assertEquals("A", tokenizer.nextToken());
897 assertEquals(1, tokenizer.nextIndex());
898 assertEquals("b", tokenizer.nextToken());
899 assertEquals(2, tokenizer.nextIndex());
900 assertEquals("c", tokenizer.nextToken());
901 assertEquals(3, tokenizer.nextIndex());
902 assertNull(tokenizer.nextToken());
903 assertEquals(3, tokenizer.nextIndex());
904 assertEquals("c", tokenizer.previousToken());
905 assertEquals(2, tokenizer.nextIndex());
906 assertEquals("b", tokenizer.previousToken());
907 assertEquals(1, tokenizer.nextIndex());
908 assertEquals("A", tokenizer.previousToken());
909 assertEquals(0, tokenizer.nextIndex());
910 assertNull(tokenizer.previousToken());
911 assertEquals(0, tokenizer.nextIndex());
912 assertEquals(-1, tokenizer.previousIndex());
913 assertEquals(3, tokenizer.size());
914 }
915 }