1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.commons.text;
19
20 import static org.junit.jupiter.api.Assertions.assertEquals;
21 import static org.junit.jupiter.api.Assertions.assertFalse;
22 import static org.junit.jupiter.api.Assertions.assertNotSame;
23 import static org.junit.jupiter.api.Assertions.assertNull;
24 import static org.junit.jupiter.api.Assertions.assertThrows;
25 import static org.junit.jupiter.api.Assertions.assertTrue;
26
27 import java.util.Arrays;
28 import java.util.Collections;
29 import java.util.List;
30 import java.util.NoSuchElementException;
31
32 import org.apache.commons.lang3.ArrayUtils;
33 import org.apache.commons.text.matcher.StringMatcher;
34 import org.apache.commons.text.matcher.StringMatcherFactory;
35 import org.junit.jupiter.api.Test;
36
37
38
39
40 class StringTokenizerTest {
41
42 private static final String CSV_SIMPLE_FIXTURE = "A,b,c";
43
44 private static final String TSV_SIMPLE_FIXTURE = "A\tb\tc";
45
46 private void checkClone(final StringTokenizer tokenizer) {
47 assertNotSame(StringTokenizer.getCSVInstance(), tokenizer);
48 assertNotSame(StringTokenizer.getTSVInstance(), tokenizer);
49 }
50
51 @Test
52 void test1() {
53
54 final String input = "a;b;c;\"d;\"\"e\";f; ; ; ";
55 final StringTokenizer tok = new StringTokenizer(input);
56 tok.setDelimiterChar(';');
57 tok.setQuoteChar('"');
58 tok.setIgnoredMatcher(StringMatcherFactory.INSTANCE.trimMatcher());
59 tok.setIgnoreEmptyTokens(false);
60 final String[] tokens = tok.getTokenArray();
61
62 final String[] expected = {"a", "b", "c", "d;\"e", "f", "", "", "" };
63
64 assertEquals(expected.length, tokens.length, Arrays.toString(tokens));
65 for (int i = 0; i < expected.length; i++) {
66 assertEquals(expected[i], tokens[i],
67 "token[" + i + "] was '" + tokens[i] + "' but was expected to be '" + expected[i] + "'");
68 }
69
70 }
71
72 @Test
73 void test2() {
74
75 final String input = "a;b;c ;\"d;\"\"e\";f; ; ;";
76 final StringTokenizer tok = new StringTokenizer(input);
77 tok.setDelimiterChar(';');
78 tok.setQuoteChar('"');
79 tok.setIgnoredMatcher(StringMatcherFactory.INSTANCE.noneMatcher());
80 tok.setIgnoreEmptyTokens(false);
81 final String[] tokens = tok.getTokenArray();
82
83 final String[] expected = {"a", "b", "c ", "d;\"e", "f", " ", " ", "" };
84
85 assertEquals(expected.length, tokens.length, Arrays.toString(tokens));
86 for (int i = 0; i < expected.length; i++) {
87 assertEquals(expected[i], tokens[i],
88 "token[" + i + "] was '" + tokens[i] + "' but was expected to be '" + expected[i] + "'");
89 }
90
91 }
92
93 @Test
94 void test3() {
95
96 final String input = "a;b; c;\"d;\"\"e\";f; ; ;";
97 final StringTokenizer tok = new StringTokenizer(input);
98 tok.setDelimiterChar(';');
99 tok.setQuoteChar('"');
100 tok.setIgnoredMatcher(StringMatcherFactory.INSTANCE.noneMatcher());
101 tok.setIgnoreEmptyTokens(false);
102 final String[] tokens = tok.getTokenArray();
103
104 final String[] expected = {"a", "b", " c", "d;\"e", "f", " ", " ", "" };
105
106 assertEquals(expected.length, tokens.length, Arrays.toString(tokens));
107 for (int i = 0; i < expected.length; i++) {
108 assertEquals(expected[i], tokens[i],
109 "token[" + i + "] was '" + tokens[i] + "' but was expected to be '" + expected[i] + "'");
110 }
111
112 }
113
114 @Test
115 void test4() {
116
117 final String input = "a;b; c;\"d;\"\"e\";f; ; ;";
118 final StringTokenizer tok = new StringTokenizer(input);
119 tok.setDelimiterChar(';');
120 tok.setQuoteChar('"');
121 tok.setIgnoredMatcher(StringMatcherFactory.INSTANCE.trimMatcher());
122 tok.setIgnoreEmptyTokens(true);
123 final String[] tokens = tok.getTokenArray();
124
125 final String[] expected = {"a", "b", "c", "d;\"e", "f" };
126
127 assertEquals(expected.length, tokens.length, Arrays.toString(tokens));
128 for (int i = 0; i < expected.length; i++) {
129 assertEquals(expected[i], tokens[i],
130 "token[" + i + "] was '" + tokens[i] + "' but was expected to be '" + expected[i] + "'");
131 }
132
133 }
134
135 @Test
136 void test5() {
137
138 final String input = "a;b; c;\"d;\"\"e\";f; ; ;";
139 final StringTokenizer tok = new StringTokenizer(input);
140 tok.setDelimiterChar(';');
141 tok.setQuoteChar('"');
142 tok.setIgnoredMatcher(StringMatcherFactory.INSTANCE.trimMatcher());
143 tok.setIgnoreEmptyTokens(false);
144 tok.setEmptyTokenAsNull(true);
145 final String[] tokens = tok.getTokenArray();
146
147 final String[] expected = {"a", "b", "c", "d;\"e", "f", null, null, null };
148
149 assertEquals(expected.length, tokens.length, Arrays.toString(tokens));
150 for (int i = 0; i < expected.length; i++) {
151 assertEquals(expected[i], tokens[i],
152 "token[" + i + "] was '" + tokens[i] + "' but was expected to be '" + expected[i] + "'");
153 }
154
155 }
156
157 @Test
158 void test6() {
159
160 final String input = "a;b; c;\"d;\"\"e\";f; ; ;";
161 final StringTokenizer tok = new StringTokenizer(input);
162 tok.setDelimiterChar(';');
163 tok.setQuoteChar('"');
164 tok.setIgnoredMatcher(StringMatcherFactory.INSTANCE.trimMatcher());
165 tok.setIgnoreEmptyTokens(false);
166
167 final String[] tokens = tok.getTokenArray();
168
169 final String[] expected = {"a", "b", " c", "d;\"e", "f", null, null, null };
170
171 int nextCount = 0;
172 while (tok.hasNext()) {
173 tok.next();
174 nextCount++;
175 }
176
177 int prevCount = 0;
178 while (tok.hasPrevious()) {
179 tok.previous();
180 prevCount++;
181 }
182
183 assertEquals(expected.length, tokens.length, Arrays.toString(tokens));
184 assertEquals(nextCount, expected.length, "could not cycle through entire token list using the 'hasNext' and 'next' methods");
185 assertEquals(prevCount, expected.length, "could not cycle through entire token list using the 'hasPrevious' and 'previous' methods");
186 }
187
188 @Test
189 void test7() {
190
191 final String input = "a b c \"d e\" f ";
192 final StringTokenizer tok = new StringTokenizer(input);
193 tok.setDelimiterMatcher(StringMatcherFactory.INSTANCE.spaceMatcher());
194 tok.setQuoteMatcher(StringMatcherFactory.INSTANCE.doubleQuoteMatcher());
195 tok.setIgnoredMatcher(StringMatcherFactory.INSTANCE.noneMatcher());
196 tok.setIgnoreEmptyTokens(false);
197 final String[] tokens = tok.getTokenArray();
198
199 final String[] expected = {"a", "", "", "b", "c", "d e", "f", "" };
200
201 assertEquals(expected.length, tokens.length, Arrays.toString(tokens));
202 for (int i = 0; i < expected.length; i++) {
203 assertEquals(expected[i], tokens[i],
204 "token[" + i + "] was '" + tokens[i] + "' but was expected to be '" + expected[i] + "'");
205 }
206
207 }
208
209 @Test
210 void test8() {
211
212 final String input = "a b c \"d e\" f ";
213 final StringTokenizer tok = new StringTokenizer(input);
214 tok.setDelimiterMatcher(StringMatcherFactory.INSTANCE.spaceMatcher());
215 tok.setQuoteMatcher(StringMatcherFactory.INSTANCE.doubleQuoteMatcher());
216 tok.setIgnoredMatcher(StringMatcherFactory.INSTANCE.noneMatcher());
217 tok.setIgnoreEmptyTokens(true);
218 final String[] tokens = tok.getTokenArray();
219
220 final String[] expected = {"a", "b", "c", "d e", "f" };
221
222 assertEquals(expected.length, tokens.length, Arrays.toString(tokens));
223 for (int i = 0; i < expected.length; i++) {
224 assertEquals(expected[i], tokens[i],
225 "token[" + i + "] was '" + tokens[i] + "' but was expected to be '" + expected[i] + "'");
226 }
227
228 }
229
230 @Test
231 void testBasic1() {
232 final String input = "a b c";
233 final StringTokenizer tok = new StringTokenizer(input);
234 assertEquals("a", tok.next());
235 assertEquals("b", tok.next());
236 assertEquals("c", tok.next());
237 assertFalse(tok.hasNext());
238 }
239
240 @Test
241 void testBasic2() {
242 final String input = "a \nb\fc";
243 final StringTokenizer tok = new StringTokenizer(input);
244 assertEquals("a", tok.next());
245 assertEquals("b", tok.next());
246 assertEquals("c", tok.next());
247 assertFalse(tok.hasNext());
248 }
249
250 @Test
251 void testBasic3() {
252 final String input = "a \nb\u0001\fc";
253 final StringTokenizer tok = new StringTokenizer(input);
254 assertEquals("a", tok.next());
255 assertEquals("b\u0001", tok.next());
256 assertEquals("c", tok.next());
257 assertFalse(tok.hasNext());
258 }
259
260 @Test
261 void testBasic4() {
262 final String input = "a \"b\" c";
263 final StringTokenizer tok = new StringTokenizer(input);
264 assertEquals("a", tok.next());
265 assertEquals("\"b\"", tok.next());
266 assertEquals("c", tok.next());
267 assertFalse(tok.hasNext());
268 }
269
270 @Test
271 void testBasic5() {
272 final String input = "a:b':c";
273 final StringTokenizer tok = new StringTokenizer(input, ':', '\'');
274 assertEquals("a", tok.next());
275 assertEquals("b'", tok.next());
276 assertEquals("c", tok.next());
277 assertFalse(tok.hasNext());
278 }
279
280 @Test
281 void testBasicDelim1() {
282 final String input = "a:b:c";
283 final StringTokenizer tok = new StringTokenizer(input, ':');
284 assertEquals("a", tok.next());
285 assertEquals("b", tok.next());
286 assertEquals("c", tok.next());
287 assertFalse(tok.hasNext());
288 }
289
290 @Test
291 void testBasicDelim2() {
292 final String input = "a:b:c";
293 final StringTokenizer tok = new StringTokenizer(input, ',');
294 assertEquals("a:b:c", tok.next());
295 assertFalse(tok.hasNext());
296 }
297
298 @Test
299 void testBasicEmpty1() {
300 final String input = "a b c";
301 final StringTokenizer tok = new StringTokenizer(input);
302 tok.setIgnoreEmptyTokens(false);
303 assertEquals("a", tok.next());
304 assertEquals("", tok.next());
305 assertEquals("b", tok.next());
306 assertEquals("c", tok.next());
307 assertFalse(tok.hasNext());
308 }
309
310 @Test
311 void testBasicEmpty2() {
312 final String input = "a b c";
313 final StringTokenizer tok = new StringTokenizer(input);
314 tok.setIgnoreEmptyTokens(false);
315 tok.setEmptyTokenAsNull(true);
316 assertEquals("a", tok.next());
317 assertNull(tok.next());
318 assertEquals("b", tok.next());
319 assertEquals("c", tok.next());
320 assertFalse(tok.hasNext());
321 }
322
323 @Test
324 void testBasicIgnoreTrimmed1() {
325 final String input = "a: bIGNOREc : ";
326 final StringTokenizer tok = new StringTokenizer(input, ':');
327 tok.setIgnoredMatcher(StringMatcherFactory.INSTANCE.stringMatcher("IGNORE"));
328 tok.setTrimmerMatcher(StringMatcherFactory.INSTANCE.trimMatcher());
329 tok.setIgnoreEmptyTokens(false);
330 tok.setEmptyTokenAsNull(true);
331 assertEquals("a", tok.next());
332 assertEquals("bc", tok.next());
333 assertNull(tok.next());
334 assertFalse(tok.hasNext());
335 }
336
337 @Test
338 void testBasicIgnoreTrimmed2() {
339 final String input = "IGNOREaIGNORE: IGNORE bIGNOREc IGNORE : IGNORE ";
340 final StringTokenizer tok = new StringTokenizer(input, ':');
341 tok.setIgnoredMatcher(StringMatcherFactory.INSTANCE.stringMatcher("IGNORE"));
342 tok.setTrimmerMatcher(StringMatcherFactory.INSTANCE.trimMatcher());
343 tok.setIgnoreEmptyTokens(false);
344 tok.setEmptyTokenAsNull(true);
345 assertEquals("a", tok.next());
346 assertEquals("bc", tok.next());
347 assertNull(tok.next());
348 assertFalse(tok.hasNext());
349 }
350
351 @Test
352 void testBasicIgnoreTrimmed3() {
353 final String input = "IGNOREaIGNORE: IGNORE bIGNOREc IGNORE : IGNORE ";
354 final StringTokenizer tok = new StringTokenizer(input, ':');
355 tok.setIgnoredMatcher(StringMatcherFactory.INSTANCE.stringMatcher("IGNORE"));
356 tok.setIgnoreEmptyTokens(false);
357 tok.setEmptyTokenAsNull(true);
358 assertEquals("a", tok.next());
359 assertEquals(" bc ", tok.next());
360 assertEquals(" ", tok.next());
361 assertFalse(tok.hasNext());
362 }
363
364 @Test
365 void testBasicIgnoreTrimmed4() {
366 final String input = "IGNOREaIGNORE: IGNORE 'bIGNOREc'IGNORE'd' IGNORE : IGNORE ";
367 final StringTokenizer tok = new StringTokenizer(input, ':', '\'');
368 tok.setIgnoredMatcher(StringMatcherFactory.INSTANCE.stringMatcher("IGNORE"));
369 tok.setTrimmerMatcher(StringMatcherFactory.INSTANCE.trimMatcher());
370 tok.setIgnoreEmptyTokens(false);
371 tok.setEmptyTokenAsNull(true);
372 assertEquals("a", tok.next());
373 assertEquals("bIGNOREcd", tok.next());
374 assertNull(tok.next());
375 assertFalse(tok.hasNext());
376 }
377
378 @Test
379 void testBasicQuoted1() {
380 final String input = "a 'b' c";
381 final StringTokenizer tok = new StringTokenizer(input, ' ', '\'');
382 assertEquals("a", tok.next());
383 assertEquals("b", tok.next());
384 assertEquals("c", tok.next());
385 assertFalse(tok.hasNext());
386 }
387
388 @Test
389 void testBasicQuoted2() {
390 final String input = "a:'b':";
391 final StringTokenizer tok = new StringTokenizer(input, ':', '\'');
392 tok.setIgnoreEmptyTokens(false);
393 tok.setEmptyTokenAsNull(true);
394 assertEquals("a", tok.next());
395 assertEquals("b", tok.next());
396 assertNull(tok.next());
397 assertFalse(tok.hasNext());
398 }
399
400 @Test
401 void testBasicQuoted3() {
402 final String input = "a:'b''c'";
403 final StringTokenizer tok = new StringTokenizer(input, ':', '\'');
404 tok.setIgnoreEmptyTokens(false);
405 tok.setEmptyTokenAsNull(true);
406 assertEquals("a", tok.next());
407 assertEquals("b'c", tok.next());
408 assertFalse(tok.hasNext());
409 }
410
411 @Test
412 void testBasicQuoted4() {
413 final String input = "a: 'b' 'c' :d";
414 final StringTokenizer tok = new StringTokenizer(input, ':', '\'');
415 tok.setTrimmerMatcher(StringMatcherFactory.INSTANCE.trimMatcher());
416 tok.setIgnoreEmptyTokens(false);
417 tok.setEmptyTokenAsNull(true);
418 assertEquals("a", tok.next());
419 assertEquals("b c", tok.next());
420 assertEquals("d", tok.next());
421 assertFalse(tok.hasNext());
422 }
423
424 @Test
425 void testBasicQuoted5() {
426 final String input = "a: 'b'x'c' :d";
427 final StringTokenizer tok = new StringTokenizer(input, ':', '\'');
428 tok.setTrimmerMatcher(StringMatcherFactory.INSTANCE.trimMatcher());
429 tok.setIgnoreEmptyTokens(false);
430 tok.setEmptyTokenAsNull(true);
431 assertEquals("a", tok.next());
432 assertEquals("bxc", tok.next());
433 assertEquals("d", tok.next());
434 assertFalse(tok.hasNext());
435 }
436
437 @Test
438 void testBasicQuoted6() {
439 final String input = "a:'b'\"c':d";
440 final StringTokenizer tok = new StringTokenizer(input, ':');
441 tok.setQuoteMatcher(StringMatcherFactory.INSTANCE.quoteMatcher());
442 assertEquals("a", tok.next());
443 assertEquals("b\"c:d", tok.next());
444 assertFalse(tok.hasNext());
445 }
446
447 @Test
448 void testBasicQuoted7() {
449 final String input = "a:\"There's a reason here\":b";
450 final StringTokenizer tok = new StringTokenizer(input, ':');
451 tok.setQuoteMatcher(StringMatcherFactory.INSTANCE.quoteMatcher());
452 assertEquals("a", tok.next());
453 assertEquals("There's a reason here", tok.next());
454 assertEquals("b", tok.next());
455 assertFalse(tok.hasNext());
456 }
457
458 @Test
459 void testBasicQuotedTrimmed1() {
460 final String input = "a: 'b' :";
461 final StringTokenizer tok = new StringTokenizer(input, ':', '\'');
462 tok.setTrimmerMatcher(StringMatcherFactory.INSTANCE.trimMatcher());
463 tok.setIgnoreEmptyTokens(false);
464 tok.setEmptyTokenAsNull(true);
465 assertEquals("a", tok.next());
466 assertEquals("b", tok.next());
467 assertNull(tok.next());
468 assertFalse(tok.hasNext());
469 }
470
471 @Test
472 void testBasicTrimmed1() {
473 final String input = "a: b : ";
474 final StringTokenizer tok = new StringTokenizer(input, ':');
475 tok.setTrimmerMatcher(StringMatcherFactory.INSTANCE.trimMatcher());
476 tok.setIgnoreEmptyTokens(false);
477 tok.setEmptyTokenAsNull(true);
478 assertEquals("a", tok.next());
479 assertEquals("b", tok.next());
480 assertNull(tok.next());
481 assertFalse(tok.hasNext());
482 }
483
484 @Test
485 void testBasicTrimmed2() {
486 final String input = "a: b :";
487 final StringTokenizer tok = new StringTokenizer(input, ':');
488 tok.setTrimmerMatcher(StringMatcherFactory.INSTANCE.stringMatcher(" "));
489 tok.setIgnoreEmptyTokens(false);
490 tok.setEmptyTokenAsNull(true);
491 assertEquals("a", tok.next());
492 assertEquals("b", tok.next());
493 assertNull(tok.next());
494 assertFalse(tok.hasNext());
495 }
496
497 @Test
498 void testChaining() {
499 final StringTokenizer tok = new StringTokenizer();
500 assertEquals(tok, tok.reset());
501 assertEquals(tok, tok.reset(""));
502 assertEquals(tok, tok.reset(ArrayUtils.EMPTY_CHAR_ARRAY));
503 assertEquals(tok, tok.setDelimiterChar(' '));
504 assertEquals(tok, tok.setDelimiterString(" "));
505 assertEquals(tok, tok.setDelimiterMatcher(null));
506 assertEquals(tok, tok.setQuoteChar(' '));
507 assertEquals(tok, tok.setQuoteMatcher(null));
508 assertEquals(tok, tok.setIgnoredChar(' '));
509 assertEquals(tok, tok.setIgnoredMatcher(null));
510 assertEquals(tok, tok.setTrimmerMatcher(null));
511 assertEquals(tok, tok.setEmptyTokenAsNull(false));
512 assertEquals(tok, tok.setIgnoreEmptyTokens(false));
513 }
514
515
516
517
518
519 @Test
520 void testCloneNotSupportedException() {
521 final Object notCloned = new StringTokenizer() {
522
523 @Override
524 Object cloneReset() throws CloneNotSupportedException {
525 throw new CloneNotSupportedException("test");
526 }
527 }.clone();
528 assertNull(notCloned);
529 }
530
531 @Test
532 void testCloneNull() {
533 final StringTokenizer tokenizer = new StringTokenizer((char[]) null);
534
535 assertNull(tokenizer.nextToken());
536 tokenizer.reset();
537 assertNull(tokenizer.nextToken());
538
539 final StringTokenizer clonedTokenizer = (StringTokenizer) tokenizer.clone();
540 tokenizer.reset();
541 assertNull(tokenizer.nextToken());
542 assertNull(clonedTokenizer.nextToken());
543 }
544
545 @Test
546 void testCloneReset() {
547 final char[] input = { 'a' };
548 final StringTokenizer tokenizer = new StringTokenizer(input);
549
550 assertEquals("a", tokenizer.nextToken());
551 tokenizer.reset(input);
552 assertEquals("a", tokenizer.nextToken());
553
554 final StringTokenizer clonedTokenizer = (StringTokenizer) tokenizer.clone();
555 input[0] = 'b';
556 tokenizer.reset(input);
557 assertEquals("b", tokenizer.nextToken());
558 assertEquals("a", clonedTokenizer.nextToken());
559 }
560
561 @Test
562 void testConstructor_charArray() {
563 StringTokenizer tok = new StringTokenizer("a b".toCharArray());
564 assertEquals("a", tok.next());
565 assertEquals("b", tok.next());
566 assertFalse(tok.hasNext());
567
568 tok = new StringTokenizer(ArrayUtils.EMPTY_CHAR_ARRAY);
569 assertFalse(tok.hasNext());
570
571 tok = new StringTokenizer((char[]) null);
572 assertFalse(tok.hasNext());
573 }
574
575 @Test
576 void testConstructor_charArray_char() {
577 StringTokenizer tok = new StringTokenizer("a b".toCharArray(), ' ');
578 assertEquals(1, tok.getDelimiterMatcher().isMatch(" ".toCharArray(), 0, 0, 1));
579 assertEquals(1, tok.getDelimiterMatcher().isMatch(" ", 0, 0, 1));
580 assertEquals("a", tok.next());
581 assertEquals("b", tok.next());
582 assertFalse(tok.hasNext());
583
584 tok = new StringTokenizer(ArrayUtils.EMPTY_CHAR_ARRAY, ' ');
585 assertFalse(tok.hasNext());
586
587 tok = new StringTokenizer((char[]) null, ' ');
588 assertFalse(tok.hasNext());
589 }
590
591 @Test
592 void testConstructor_charArray_char_char() {
593 StringTokenizer tok = new StringTokenizer("a b".toCharArray(), ' ', '"');
594 assertEquals(1, tok.getDelimiterMatcher().isMatch(" ".toCharArray(), 0, 0, 1));
595 assertEquals(1, tok.getDelimiterMatcher().isMatch(" ", 0, 0, 1));
596 assertEquals(1, tok.getQuoteMatcher().isMatch("\"".toCharArray(), 0, 0, 1));
597 assertEquals(1, tok.getQuoteMatcher().isMatch("\"", 0, 0, 1));
598 assertEquals("a", tok.next());
599 assertEquals("b", tok.next());
600 assertFalse(tok.hasNext());
601
602 tok = new StringTokenizer(ArrayUtils.EMPTY_CHAR_ARRAY, ' ', '"');
603 assertFalse(tok.hasNext());
604
605 tok = new StringTokenizer((char[]) null, ' ', '"');
606 assertFalse(tok.hasNext());
607 }
608
609 @Test
610 void testConstructor_String() {
611 StringTokenizer tok = new StringTokenizer("a b");
612 assertEquals("a", tok.next());
613 assertEquals("b", tok.next());
614 assertFalse(tok.hasNext());
615
616 tok = new StringTokenizer("");
617 assertFalse(tok.hasNext());
618
619 tok = new StringTokenizer((String) null);
620 assertFalse(tok.hasNext());
621 }
622
623 @Test
624 void testConstructor_String_char() {
625 StringTokenizer tok = new StringTokenizer("a b", ' ');
626 assertEquals(1, tok.getDelimiterMatcher().isMatch(" ".toCharArray(), 0, 0, 1));
627 assertEquals(1, tok.getDelimiterMatcher().isMatch(" ", 0, 0, 1));
628 assertEquals("a", tok.next());
629 assertEquals("b", tok.next());
630 assertFalse(tok.hasNext());
631
632 tok = new StringTokenizer("", ' ');
633 assertFalse(tok.hasNext());
634
635 tok = new StringTokenizer((String) null, ' ');
636 assertFalse(tok.hasNext());
637 }
638
639 @Test
640 void testConstructor_String_char_char() {
641 StringTokenizer tok = new StringTokenizer("a b", ' ', '"');
642 assertEquals(1, tok.getDelimiterMatcher().isMatch(" ".toCharArray(), 0, 0, 1));
643 assertEquals(1, tok.getDelimiterMatcher().isMatch(" ", 0, 0, 1));
644 assertEquals(1, tok.getQuoteMatcher().isMatch("\"".toCharArray(), 0, 0, 1));
645 assertEquals(1, tok.getQuoteMatcher().isMatch("\"", 0, 0, 1));
646 assertEquals("a", tok.next());
647 assertEquals("b", tok.next());
648 assertFalse(tok.hasNext());
649
650 tok = new StringTokenizer("", ' ', '"');
651 assertFalse(tok.hasNext());
652
653 tok = new StringTokenizer((String) null, ' ', '"');
654 assertFalse(tok.hasNext());
655 }
656
657 private void testCSV(final String data) {
658 testXSVAbc(StringTokenizer.getCSVInstance(data));
659 testXSVAbc(StringTokenizer.getCSVInstance(data.toCharArray()));
660 }
661
662 @Test
663 void testCSVEmpty() {
664 testEmpty(StringTokenizer.getCSVInstance());
665 testEmpty(StringTokenizer.getCSVInstance(""));
666 }
667
668 @Test
669 void testCSVSimple() {
670 testCSV(CSV_SIMPLE_FIXTURE);
671 }
672
673 @Test
674 void testCSVSimpleNeedsTrim() {
675 testCSV(" " + CSV_SIMPLE_FIXTURE);
676 testCSV(" \n\t " + CSV_SIMPLE_FIXTURE);
677 testCSV(" \n " + CSV_SIMPLE_FIXTURE + "\n\n\r");
678 }
679
680 @Test
681 void testDelimMatcher() {
682 final String input = "a/b\\c";
683 final StringMatcher delimMatcher = StringMatcherFactory.INSTANCE.charSetMatcher('/', '\\');
684
685 final StringTokenizer tok = new StringTokenizer(input, delimMatcher);
686 assertEquals("a", tok.next());
687 assertEquals("b", tok.next());
688 assertEquals("c", tok.next());
689 assertFalse(tok.hasNext());
690 }
691
692 @Test
693 void testDelimMatcherQuoteMatcher() {
694 final String input = "`a`;`b`;`c`";
695 final StringMatcher delimMatcher = StringMatcherFactory.INSTANCE.charSetMatcher(';');
696 final StringMatcher quoteMatcher = StringMatcherFactory.INSTANCE.charSetMatcher('`');
697
698 final StringTokenizer tok = new StringTokenizer(input, delimMatcher, quoteMatcher);
699 assertEquals("a", tok.next());
700 assertEquals("b", tok.next());
701 assertEquals("c", tok.next());
702 assertFalse(tok.hasNext());
703 }
704
705 @Test
706 void testDelimString() {
707 final String input = "a##b##c";
708 final StringTokenizer tok = new StringTokenizer(input, "##");
709
710 assertEquals("a", tok.next());
711 assertEquals("b", tok.next());
712 assertEquals("c", tok.next());
713 assertFalse(tok.hasNext());
714 }
715
716 void testEmpty(final StringTokenizer tokenizer) {
717 checkClone(tokenizer);
718 assertFalse(tokenizer.hasNext());
719 assertFalse(tokenizer.hasPrevious());
720 assertNull(tokenizer.nextToken());
721 assertEquals(0, tokenizer.size());
722 assertThrows(NoSuchElementException.class, tokenizer::next);
723 }
724
725 @Test
726 void testGetContent() {
727 final String input = "a b c \"d e\" f ";
728 StringTokenizer tok = new StringTokenizer(input);
729 assertEquals(input, tok.getContent());
730
731 tok = new StringTokenizer(input.toCharArray());
732 assertEquals(input, tok.getContent());
733
734 tok = new StringTokenizer();
735 assertNull(tok.getContent());
736 }
737
738 @Test
739 void testIteration() {
740 final StringTokenizer tkn = new StringTokenizer("a b c");
741 assertFalse(tkn.hasPrevious());
742 assertThrows(NoSuchElementException.class, tkn::previous);
743 assertTrue(tkn.hasNext());
744
745 assertEquals("a", tkn.next());
746 assertThrows(UnsupportedOperationException.class, tkn::remove);
747 assertThrows(UnsupportedOperationException.class, () -> tkn.set("x"));
748 assertThrows(UnsupportedOperationException.class, () -> tkn.add("y"));
749 assertTrue(tkn.hasPrevious());
750 assertTrue(tkn.hasNext());
751
752 assertEquals("b", tkn.next());
753 assertTrue(tkn.hasPrevious());
754 assertTrue(tkn.hasNext());
755
756 assertEquals("c", tkn.next());
757 assertTrue(tkn.hasPrevious());
758 assertFalse(tkn.hasNext());
759
760 assertThrows(NoSuchElementException.class, tkn::next);
761 assertTrue(tkn.hasPrevious());
762 assertFalse(tkn.hasNext());
763 }
764
765 @Test
766 void testListArray() {
767 final String input = "a b c";
768 final StringTokenizer tok = new StringTokenizer(input);
769 final String[] array = tok.getTokenArray();
770 final List<String> list = tok.getTokenList();
771
772 assertEquals(Arrays.asList(array), list);
773 assertEquals(3, list.size());
774
775
776 list.set(0, "z");
777 list.remove(1);
778 list.set(1, "y");
779 list.add("x");
780
781 assertEquals(Arrays.asList("z", "y", "x"), list);
782
783
784 assertEquals(Arrays.asList(array), tok.getTokenList());
785 assertEquals("a", tok.next());
786 assertEquals("b", tok.next());
787 assertEquals("c", tok.next());
788 }
789
790 @Test
791 void testPreviousTokenAndSetEmptyTokenAsNull() {
792 final StringTokenizer strTokenizer = StringTokenizer.getTSVInstance(" \t\n\r\f");
793 strTokenizer.setEmptyTokenAsNull(true);
794
795 assertNull(strTokenizer.previousToken());
796 }
797
798 @Test
799 void testReset() {
800 final StringTokenizer tok = new StringTokenizer("a b c");
801 assertEquals("a", tok.next());
802 assertEquals("b", tok.next());
803 assertEquals("c", tok.next());
804 assertFalse(tok.hasNext());
805
806 tok.reset();
807 assertEquals("a", tok.next());
808 assertEquals("b", tok.next());
809 assertEquals("c", tok.next());
810 assertFalse(tok.hasNext());
811 }
812
813 @Test
814 void testReset_charArray() {
815 final StringTokenizer tok = new StringTokenizer("x x x");
816
817 final char[] array = {'a', 'b', 'c' };
818 tok.reset(array);
819 assertEquals("abc", tok.next());
820 assertFalse(tok.hasNext());
821
822 tok.reset((char[]) null);
823 assertFalse(tok.hasNext());
824 }
825
826 @Test
827 void testReset_String() {
828 final StringTokenizer tok = new StringTokenizer("x x x");
829 tok.reset("d e");
830 assertEquals("d", tok.next());
831 assertEquals("e", tok.next());
832 assertFalse(tok.hasNext());
833
834 tok.reset((String) null);
835 assertFalse(tok.hasNext());
836 }
837
838 @Test
839 void testStringTokenizerQuoteMatcher() {
840 final char[] chars = {'\'', 'a', 'c', '\'', 'd' };
841 final StringTokenizer tokens = new StringTokenizer(chars, StringMatcherFactory.INSTANCE.commaMatcher(),
842 StringMatcherFactory.INSTANCE.quoteMatcher());
843 assertEquals("acd", tokens.next());
844 }
845
846 @Test
847 void testStringTokenizerStringMatcher() {
848 final char[] chars = {'a', 'b', 'c', 'd' };
849 final StringTokenizer tokens = new StringTokenizer(chars, "bc");
850 assertEquals("a", tokens.next());
851 assertEquals("d", tokens.next());
852 }
853
854 @Test
855 void testStringTokenizerStrMatcher() {
856 final char[] chars = {'a', ',', 'c' };
857 final StringTokenizer tokens = new StringTokenizer(chars, StringMatcherFactory.INSTANCE.commaMatcher());
858 assertEquals("a", tokens.next());
859 assertEquals("c", tokens.next());
860 }
861
862 @Test
863 void testTokenizeSubclassInputChange() {
864 final StringTokenizer tkn = new StringTokenizer("a b c d e") {
865
866 @Override
867 protected List<String> tokenize(final char[] chars, final int offset, final int count) {
868 return super.tokenize("w x y z".toCharArray(), 2, 5);
869 }
870 };
871 assertEquals("x", tkn.next());
872 assertEquals("y", tkn.next());
873 }
874
875 @Test
876 void testTokenizeSubclassOutputChange() {
877 final StringTokenizer tkn = new StringTokenizer("a b c") {
878
879 @Override
880 protected List<String> tokenize(final char[] chars, final int offset, final int count) {
881 final List<String> list = super.tokenize(chars, offset, count);
882 Collections.reverse(list);
883 return list;
884 }
885 };
886 assertEquals("c", tkn.next());
887 assertEquals("b", tkn.next());
888 assertEquals("a", tkn.next());
889 }
890
891 @Test
892 void testToString() {
893 final StringTokenizer tkn = new StringTokenizer("a b c d e");
894 assertEquals("StringTokenizer[not tokenized yet]", tkn.toString());
895 tkn.next();
896 assertEquals("StringTokenizer[a, b, c, d, e]", tkn.toString());
897 }
898
899 @Test
900 void testTSV() {
901 testXSVAbc(StringTokenizer.getTSVInstance(TSV_SIMPLE_FIXTURE));
902 testXSVAbc(StringTokenizer.getTSVInstance(TSV_SIMPLE_FIXTURE.toCharArray()));
903 }
904
905 @Test
906 void testTSVEmpty() {
907 testEmpty(StringTokenizer.getTSVInstance());
908 testEmpty(StringTokenizer.getTSVInstance(""));
909 }
910
911 void testXSVAbc(final StringTokenizer tokenizer) {
912 checkClone(tokenizer);
913 assertEquals(-1, tokenizer.previousIndex());
914 assertEquals(0, tokenizer.nextIndex());
915 assertNull(tokenizer.previousToken());
916 assertEquals("A", tokenizer.nextToken());
917 assertEquals(1, tokenizer.nextIndex());
918 assertEquals("b", tokenizer.nextToken());
919 assertEquals(2, tokenizer.nextIndex());
920 assertEquals("c", tokenizer.nextToken());
921 assertEquals(3, tokenizer.nextIndex());
922 assertNull(tokenizer.nextToken());
923 assertEquals(3, tokenizer.nextIndex());
924 assertEquals("c", tokenizer.previousToken());
925 assertEquals(2, tokenizer.nextIndex());
926 assertEquals("b", tokenizer.previousToken());
927 assertEquals(1, tokenizer.nextIndex());
928 assertEquals("A", tokenizer.previousToken());
929 assertEquals(0, tokenizer.nextIndex());
930 assertNull(tokenizer.previousToken());
931 assertEquals(0, tokenizer.nextIndex());
932 assertEquals(-1, tokenizer.previousIndex());
933 assertEquals(3, tokenizer.size());
934 }
935 }