1 /* 2 * Licensed to the Apache Software Foundation (ASF) under one or more 3 * contributor license agreements. See the NOTICE file distributed with 4 * this work for additional information regarding copyright ownership. 5 * The ASF licenses this file to You under the Apache License, Version 2.0 6 * (the "License"); you may not use this file except in compliance with 7 * the License. You may obtain a copy of the License at 8 * 9 * http://www.apache.org/licenses/LICENSE-2.0 10 * 11 * Unless required by applicable law or agreed to in writing, software 12 * distributed under the License is distributed on an "AS IS" BASIS, 13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 * See the License for the specific language governing permissions and 15 * limitations under the License. 16 */ 17 package org.apache.commons.text; 18 19 import java.util.ArrayList; 20 import java.util.Collections; 21 import java.util.List; 22 import java.util.ListIterator; 23 import java.util.NoSuchElementException; 24 25 import org.apache.commons.lang3.ArrayUtils; 26 import org.apache.commons.lang3.StringUtils; 27 28 /** 29 * Tokenizes a string based on delimiters (separators) 30 * and supporting quoting and ignored character concepts. 31 * <p> 32 * This class can split a String into many smaller strings. It aims 33 * to do a similar job to {@link java.util.StringTokenizer StringTokenizer}, 34 * however it offers much more control and flexibility including implementing 35 * the {@code ListIterator} interface. By default, it is set up 36 * like {@code StringTokenizer}. 37 * <p> 38 * The input String is split into a number of <i>tokens</i>. 39 * Each token is separated from the next String by a <i>delimiter</i>. 40 * One or more delimiter characters must be specified. 41 * <p> 42 * Each token may be surrounded by quotes. 43 * The <i>quote</i> matcher specifies the quote character(s). 44 * A quote may be escaped within a quoted section by duplicating itself. 45 * <p> 46 * Between each token and the delimiter are potentially characters that need trimming. 47 * The <i>trimmer</i> matcher specifies these characters. 48 * One usage might be to trim whitespace characters. 49 * <p> 50 * At any point outside the quotes there might potentially be invalid characters. 51 * The <i>ignored</i> matcher specifies these characters to be removed. 52 * One usage might be to remove new line characters. 53 * <p> 54 * Empty tokens may be removed or returned as null. 55 * <pre> 56 * "a,b,c" - Three tokens "a","b","c" (comma delimiter) 57 * " a, b , c " - Three tokens "a","b","c" (default CSV processing trims whitespace) 58 * "a, ", b ,", c" - Three tokens "a, " , " b ", ", c" (quoted text untouched) 59 * </pre> 60 * 61 * <table> 62 * <caption>StrTokenizer properties and options</caption> 63 * <tr> 64 * <th>Property</th><th>Type</th><th>Default</th> 65 * </tr> 66 * <tr> 67 * <td>delim</td><td>CharSetMatcher</td><td>{ \t\n\r\f}</td> 68 * </tr> 69 * <tr> 70 * <td>quote</td><td>NoneMatcher</td><td>{}</td> 71 * </tr> 72 * <tr> 73 * <td>ignore</td><td>NoneMatcher</td><td>{}</td> 74 * </tr> 75 * <tr> 76 * <td>emptyTokenAsNull</td><td>boolean</td><td>false</td> 77 * </tr> 78 * <tr> 79 * <td>ignoreEmptyTokens</td><td>boolean</td><td>true</td> 80 * </tr> 81 * </table> 82 * 83 * @since 1.0 84 * @deprecated Deprecated as of 1.3, use {@link StringTokenizer} instead. This class will be removed in 2.0. 85 */ 86 @Deprecated 87 public class StrTokenizer implements ListIterator<String>, Cloneable { 88 89 /** Comma separated values tokenizer internal variable. */ 90 private static final StrTokenizer CSV_TOKENIZER_PROTOTYPE; 91 92 /** Tab separated values tokenizer internal variable. */ 93 private static final StrTokenizer TSV_TOKENIZER_PROTOTYPE; 94 95 static { 96 CSV_TOKENIZER_PROTOTYPE = new StrTokenizer(); 97 CSV_TOKENIZER_PROTOTYPE.setDelimiterMatcher(StrMatcher.commaMatcher()); 98 CSV_TOKENIZER_PROTOTYPE.setQuoteMatcher(StrMatcher.doubleQuoteMatcher()); 99 CSV_TOKENIZER_PROTOTYPE.setIgnoredMatcher(StrMatcher.noneMatcher()); 100 CSV_TOKENIZER_PROTOTYPE.setTrimmerMatcher(StrMatcher.trimMatcher()); 101 CSV_TOKENIZER_PROTOTYPE.setEmptyTokenAsNull(false); 102 CSV_TOKENIZER_PROTOTYPE.setIgnoreEmptyTokens(false); 103 104 TSV_TOKENIZER_PROTOTYPE = new StrTokenizer(); 105 TSV_TOKENIZER_PROTOTYPE.setDelimiterMatcher(StrMatcher.tabMatcher()); 106 TSV_TOKENIZER_PROTOTYPE.setQuoteMatcher(StrMatcher.doubleQuoteMatcher()); 107 TSV_TOKENIZER_PROTOTYPE.setIgnoredMatcher(StrMatcher.noneMatcher()); 108 TSV_TOKENIZER_PROTOTYPE.setTrimmerMatcher(StrMatcher.trimMatcher()); 109 TSV_TOKENIZER_PROTOTYPE.setEmptyTokenAsNull(false); 110 TSV_TOKENIZER_PROTOTYPE.setIgnoreEmptyTokens(false); 111 } 112 113 /** 114 * Returns a clone of {@code CSV_TOKENIZER_PROTOTYPE}. 115 * 116 * @return a clone of {@code CSV_TOKENIZER_PROTOTYPE}. 117 */ 118 private static StrTokenizer getCSVClone() { 119 return (StrTokenizer) CSV_TOKENIZER_PROTOTYPE.clone(); 120 } 121 122 /** 123 * Gets a new tokenizer instance which parses Comma Separated Value strings 124 * initializing it with the given input. The default for CSV processing 125 * will be trim whitespace from both ends (which can be overridden with 126 * the setTrimmer method). 127 * <p> 128 * You must call a "reset" method to set the string which you want to parse. 129 * </p> 130 * @return a new tokenizer instance which parses Comma Separated Value strings 131 */ 132 public static StrTokenizer getCSVInstance() { 133 return getCSVClone(); 134 } 135 136 /** 137 * Gets a new tokenizer instance which parses Comma Separated Value strings 138 * initializing it with the given input. The default for CSV processing 139 * will be trim whitespace from both ends (which can be overridden with 140 * the setTrimmer method). 141 * 142 * @param input the text to parse 143 * @return a new tokenizer instance which parses Comma Separated Value strings 144 */ 145 public static StrTokenizer getCSVInstance(final char[] input) { 146 final StrTokenizer tok = getCSVClone(); 147 tok.reset(input); 148 return tok; 149 } 150 151 /** 152 * Gets a new tokenizer instance which parses Comma Separated Value strings 153 * initializing it with the given input. The default for CSV processing 154 * will be trim whitespace from both ends (which can be overridden with 155 * the setTrimmer method). 156 * 157 * @param input the text to parse 158 * @return a new tokenizer instance which parses Comma Separated Value strings 159 */ 160 public static StrTokenizer getCSVInstance(final String input) { 161 final StrTokenizer tok = getCSVClone(); 162 tok.reset(input); 163 return tok; 164 } 165 /** 166 * Returns a clone of {@code TSV_TOKENIZER_PROTOTYPE}. 167 * 168 * @return a clone of {@code TSV_TOKENIZER_PROTOTYPE}. 169 */ 170 private static StrTokenizer getTSVClone() { 171 return (StrTokenizer) TSV_TOKENIZER_PROTOTYPE.clone(); 172 } 173 174 /** 175 * Gets a new tokenizer instance which parses Tab Separated Value strings. 176 * The default for CSV processing will be trim whitespace from both ends 177 * (which can be overridden with the setTrimmer method). 178 * <p> 179 * You must call a "reset" method to set the string which you want to parse. 180 * </p> 181 * @return a new tokenizer instance which parses Tab Separated Value strings. 182 */ 183 public static StrTokenizer getTSVInstance() { 184 return getTSVClone(); 185 } 186 187 /** 188 * Gets a new tokenizer instance which parses Tab Separated Value strings. 189 * The default for CSV processing will be trim whitespace from both ends 190 * (which can be overridden with the setTrimmer method). 191 * @param input the string to parse 192 * @return a new tokenizer instance which parses Tab Separated Value strings. 193 */ 194 public static StrTokenizer getTSVInstance(final char[] input) { 195 final StrTokenizer tok = getTSVClone(); 196 tok.reset(input); 197 return tok; 198 } 199 200 /** 201 * Gets a new tokenizer instance which parses Tab Separated Value strings. 202 * The default for CSV processing will be trim whitespace from both ends 203 * (which can be overridden with the setTrimmer method). 204 * @param input the string to parse 205 * @return a new tokenizer instance which parses Tab Separated Value strings. 206 */ 207 public static StrTokenizer getTSVInstance(final String input) { 208 final StrTokenizer tok = getTSVClone(); 209 tok.reset(input); 210 return tok; 211 } 212 213 /** The text to work on. */ 214 private char[] chars; 215 216 /** The parsed tokens. */ 217 private String[] tokens; 218 219 /** The current iteration position. */ 220 private int tokenPos; 221 222 /** The delimiter matcher. */ 223 private StrMatcher delimMatcher = StrMatcher.splitMatcher(); 224 225 /** The quote matcher. */ 226 private StrMatcher quoteMatcher = StrMatcher.noneMatcher(); 227 228 /** The ignored matcher. */ 229 private StrMatcher ignoredMatcher = StrMatcher.noneMatcher(); 230 231 /** The trimmer matcher. */ 232 private StrMatcher trimmerMatcher = StrMatcher.noneMatcher(); 233 234 /** Whether to return empty tokens as null. */ 235 private boolean emptyAsNull; 236 237 /** Whether to ignore empty tokens. */ 238 private boolean ignoreEmptyTokens = true; 239 240 /** 241 * Constructs a tokenizer splitting on space, tab, newline and form feed 242 * as per StringTokenizer, but with no text to tokenize. 243 * <p> 244 * This constructor is normally used with {@link #reset(String)}. 245 * </p> 246 */ 247 public StrTokenizer() { 248 this.chars = null; 249 } 250 251 /** 252 * Constructs a tokenizer splitting on space, tab, newline and form feed 253 * as per StringTokenizer. 254 * 255 * @param input the string which is to be parsed, not cloned 256 */ 257 public StrTokenizer(final char[] input) { 258 if (input == null) { 259 this.chars = null; 260 } else { 261 this.chars = input.clone(); 262 } 263 } 264 265 /** 266 * Constructs a tokenizer splitting on the specified character. 267 * 268 * @param input the string which is to be parsed, not cloned 269 * @param delim the field delimiter character 270 */ 271 public StrTokenizer(final char[] input, final char delim) { 272 this(input); 273 setDelimiterChar(delim); 274 } 275 276 /** 277 * Constructs a tokenizer splitting on the specified delimiter character 278 * and handling quotes using the specified quote character. 279 * 280 * @param input the string which is to be parsed, not cloned 281 * @param delim the field delimiter character 282 * @param quote the field quoted string character 283 */ 284 public StrTokenizer(final char[] input, final char delim, final char quote) { 285 this(input, delim); 286 setQuoteChar(quote); 287 } 288 289 /** 290 * Constructs a tokenizer splitting on the specified string. 291 * 292 * @param input the string which is to be parsed, not cloned 293 * @param delim the field delimiter string 294 */ 295 public StrTokenizer(final char[] input, final String delim) { 296 this(input); 297 setDelimiterString(delim); 298 } 299 300 /** 301 * Constructs a tokenizer splitting using the specified delimiter matcher. 302 * 303 * @param input the string which is to be parsed, not cloned 304 * @param delim the field delimiter matcher 305 */ 306 public StrTokenizer(final char[] input, final StrMatcher delim) { 307 this(input); 308 setDelimiterMatcher(delim); 309 } 310 311 /** 312 * Constructs a tokenizer splitting using the specified delimiter matcher 313 * and handling quotes using the specified quote matcher. 314 * 315 * @param input the string which is to be parsed, not cloned 316 * @param delim the field delimiter character 317 * @param quote the field quoted string character 318 */ 319 public StrTokenizer(final char[] input, final StrMatcher delim, final StrMatcher quote) { 320 this(input, delim); 321 setQuoteMatcher(quote); 322 } 323 324 /** 325 * Constructs a tokenizer splitting on space, tab, newline and form feed 326 * as per StringTokenizer. 327 * 328 * @param input the string which is to be parsed 329 */ 330 public StrTokenizer(final String input) { 331 if (input != null) { 332 chars = input.toCharArray(); 333 } else { 334 chars = null; 335 } 336 } 337 338 /** 339 * Constructs a tokenizer splitting on the specified delimiter character. 340 * 341 * @param input the string which is to be parsed 342 * @param delim the field delimiter character 343 */ 344 public StrTokenizer(final String input, final char delim) { 345 this(input); 346 setDelimiterChar(delim); 347 } 348 349 /** 350 * Constructs a tokenizer splitting on the specified delimiter character 351 * and handling quotes using the specified quote character. 352 * 353 * @param input the string which is to be parsed 354 * @param delim the field delimiter character 355 * @param quote the field quoted string character 356 */ 357 public StrTokenizer(final String input, final char delim, final char quote) { 358 this(input, delim); 359 setQuoteChar(quote); 360 } 361 362 /** 363 * Constructs a tokenizer splitting on the specified delimiter string. 364 * 365 * @param input the string which is to be parsed 366 * @param delim the field delimiter string 367 */ 368 public StrTokenizer(final String input, final String delim) { 369 this(input); 370 setDelimiterString(delim); 371 } 372 373 /** 374 * Constructs a tokenizer splitting using the specified delimiter matcher. 375 * 376 * @param input the string which is to be parsed 377 * @param delim the field delimiter matcher 378 */ 379 public StrTokenizer(final String input, final StrMatcher delim) { 380 this(input); 381 setDelimiterMatcher(delim); 382 } 383 384 /** 385 * Constructs a tokenizer splitting using the specified delimiter matcher 386 * and handling quotes using the specified quote matcher. 387 * 388 * @param input the string which is to be parsed 389 * @param delim the field delimiter matcher 390 * @param quote the field quoted string matcher 391 */ 392 public StrTokenizer(final String input, final StrMatcher delim, final StrMatcher quote) { 393 this(input, delim); 394 setQuoteMatcher(quote); 395 } 396 397 /** 398 * Unsupported ListIterator operation. 399 * @param obj this parameter ignored. 400 * @throws UnsupportedOperationException always 401 */ 402 @Override 403 public void add(final String obj) { 404 throw new UnsupportedOperationException("add() is unsupported"); 405 } 406 407 /** 408 * Adds a token to a list, paying attention to the parameters we've set. 409 * 410 * @param list the list to add to 411 * @param tok the token to add 412 */ 413 private void addToken(final List<String> list, String tok) { 414 if (tok == null || tok.isEmpty()) { 415 if (isIgnoreEmptyTokens()) { 416 return; 417 } 418 if (isEmptyTokenAsNull()) { 419 tok = null; 420 } 421 } 422 list.add(tok); 423 } 424 425 /** 426 * Checks if tokenization has been done, and if not then do it. 427 */ 428 private void checkTokenized() { 429 if (tokens == null) { 430 if (chars == null) { 431 // still call tokenize as subclass may do some work 432 final List<String> split = tokenize(null, 0, 0); 433 tokens = split.toArray(ArrayUtils.EMPTY_STRING_ARRAY); 434 } else { 435 final List<String> split = tokenize(chars, 0, chars.length); 436 tokens = split.toArray(ArrayUtils.EMPTY_STRING_ARRAY); 437 } 438 } 439 } 440 441 /** 442 * Creates a new instance of this Tokenizer. The new instance is reset so 443 * that it will be at the start of the token list. 444 * If a {@link CloneNotSupportedException} is caught, return {@code null}. 445 * 446 * @return a new instance of this Tokenizer which has been reset. 447 */ 448 @Override 449 public Object clone() { 450 try { 451 return cloneReset(); 452 } catch (final CloneNotSupportedException ex) { 453 return null; 454 } 455 } 456 457 /** 458 * Creates a new instance of this Tokenizer. The new instance is reset so that 459 * it will be at the start of the token list. 460 * 461 * @return a new instance of this Tokenizer which has been reset. 462 * @throws CloneNotSupportedException if there is a problem cloning 463 */ 464 Object cloneReset() throws CloneNotSupportedException { 465 // this method exists to enable 100% test coverage 466 final StrTokenizer cloned = (StrTokenizer) super.clone(); 467 if (cloned.chars != null) { 468 cloned.chars = cloned.chars.clone(); 469 } 470 cloned.reset(); 471 return cloned; 472 } 473 474 /** 475 * Gets the String content that the tokenizer is parsing. 476 * 477 * @return The string content being parsed 478 */ 479 public String getContent() { 480 if (chars == null) { 481 return null; 482 } 483 return new String(chars); 484 } 485 486 /** 487 * Gets the field delimiter matcher. 488 * 489 * @return The delimiter matcher in use 490 */ 491 public StrMatcher getDelimiterMatcher() { 492 return this.delimMatcher; 493 } 494 495 /** 496 * Gets the ignored character matcher. 497 * <p> 498 * These characters are ignored when parsing the String, unless they are 499 * within a quoted region. 500 * The default value is not to ignore anything. 501 * </p> 502 * 503 * @return The ignored matcher in use 504 */ 505 public StrMatcher getIgnoredMatcher() { 506 return ignoredMatcher; 507 } 508 509 /** 510 * Gets the quote matcher currently in use. 511 * <p> 512 * The quote character is used to wrap data between the tokens. 513 * This enables delimiters to be entered as data. 514 * The default value is '"' (double quote). 515 * </p> 516 * 517 * @return The quote matcher in use 518 */ 519 public StrMatcher getQuoteMatcher() { 520 return quoteMatcher; 521 } 522 523 /** 524 * Gets a copy of the full token list as an independent modifiable array. 525 * 526 * @return The tokens as a String array 527 */ 528 public String[] getTokenArray() { 529 checkTokenized(); 530 return tokens.clone(); 531 } 532 533 /** 534 * Gets a copy of the full token list as an independent modifiable list. 535 * 536 * @return The tokens as a String array 537 */ 538 public List<String> getTokenList() { 539 checkTokenized(); 540 final List<String> list = new ArrayList<>(tokens.length); 541 Collections.addAll(list, tokens); 542 543 return list; 544 } 545 546 /** 547 * Gets the trimmer character matcher. 548 * <p> 549 * These characters are trimmed off on each side of the delimiter 550 * until the token or quote is found. 551 * The default value is not to trim anything. 552 * </p> 553 * 554 * @return The trimmer matcher in use 555 */ 556 public StrMatcher getTrimmerMatcher() { 557 return trimmerMatcher; 558 } 559 560 /** 561 * Checks whether there are any more tokens. 562 * 563 * @return true if there are more tokens 564 */ 565 @Override 566 public boolean hasNext() { 567 checkTokenized(); 568 return tokenPos < tokens.length; 569 } 570 571 /** 572 * Checks whether there are any previous tokens that can be iterated to. 573 * 574 * @return true if there are previous tokens 575 */ 576 @Override 577 public boolean hasPrevious() { 578 checkTokenized(); 579 return tokenPos > 0; 580 } 581 582 /** 583 * Gets whether the tokenizer currently returns empty tokens as null. 584 * The default for this property is false. 585 * 586 * @return true if empty tokens are returned as null 587 */ 588 public boolean isEmptyTokenAsNull() { 589 return this.emptyAsNull; 590 } 591 592 /** 593 * Gets whether the tokenizer currently ignores empty tokens. 594 * The default for this property is true. 595 * 596 * @return true if empty tokens are not returned 597 */ 598 public boolean isIgnoreEmptyTokens() { 599 return ignoreEmptyTokens; 600 } 601 602 /** 603 * Checks if the characters at the index specified match the quote 604 * already matched in readNextToken(). 605 * 606 * @param srcChars the character array being tokenized 607 * @param pos the position to check for a quote 608 * @param len the length of the character array being tokenized 609 * @param quoteStart the start position of the matched quote, 0 if no quoting 610 * @param quoteLen the length of the matched quote, 0 if no quoting 611 * @return true if a quote is matched 612 */ 613 private boolean isQuote(final char[] srcChars, 614 final int pos, 615 final int len, 616 final int quoteStart, 617 final int quoteLen) { 618 for (int i = 0; i < quoteLen; i++) { 619 if (pos + i >= len || srcChars[pos + i] != srcChars[quoteStart + i]) { 620 return false; 621 } 622 } 623 return true; 624 } 625 626 /** 627 * Gets the next token. 628 * 629 * @return The next String token 630 * @throws NoSuchElementException if there are no more elements 631 */ 632 @Override 633 public String next() { 634 if (hasNext()) { 635 return tokens[tokenPos++]; 636 } 637 throw new NoSuchElementException(); 638 } 639 640 /** 641 * Gets the index of the next token to return. 642 * 643 * @return The next token index 644 */ 645 @Override 646 public int nextIndex() { 647 return tokenPos; 648 } 649 650 /** 651 * Gets the next token from the String. 652 * Equivalent to {@link #next()} except it returns null rather than 653 * throwing {@link NoSuchElementException} when no tokens remain. 654 * 655 * @return The next sequential token, or null when no more tokens are found 656 */ 657 public String nextToken() { 658 if (hasNext()) { 659 return tokens[tokenPos++]; 660 } 661 return null; 662 } 663 664 /** 665 * Gets the token previous to the last returned token. 666 * 667 * @return The previous token 668 */ 669 @Override 670 public String previous() { 671 if (hasPrevious()) { 672 return tokens[--tokenPos]; 673 } 674 throw new NoSuchElementException(); 675 } 676 677 /** 678 * Gets the index of the previous token. 679 * 680 * @return The previous token index 681 */ 682 @Override 683 public int previousIndex() { 684 return tokenPos - 1; 685 } 686 687 /** 688 * Gets the previous token from the String. 689 * 690 * @return The previous sequential token, or null when no more tokens are found 691 */ 692 public String previousToken() { 693 if (hasPrevious()) { 694 return tokens[--tokenPos]; 695 } 696 return null; 697 } 698 699 /** 700 * Reads character by character through the String to get the next token. 701 * 702 * @param srcChars the character array being tokenized 703 * @param start the first character of field 704 * @param len the length of the character array being tokenized 705 * @param workArea a temporary work area 706 * @param tokenList the list of parsed tokens 707 * @return The starting position of the next field (the character 708 * immediately after the delimiter), or -1 if end of string found 709 */ 710 private int readNextToken(final char[] srcChars, 711 int start, 712 final int len, 713 final StrBuilder workArea, 714 final List<String> tokenList) { 715 // skip all leading whitespace, unless it is the 716 // field delimiter or the quote character 717 while (start < len) { 718 final int removeLen = Math.max( 719 getIgnoredMatcher().isMatch(srcChars, start, start, len), 720 getTrimmerMatcher().isMatch(srcChars, start, start, len)); 721 if (removeLen == 0 722 || getDelimiterMatcher().isMatch(srcChars, start, start, len) > 0 723 || getQuoteMatcher().isMatch(srcChars, start, start, len) > 0) { 724 break; 725 } 726 start += removeLen; 727 } 728 729 // handle reaching end 730 if (start >= len) { 731 addToken(tokenList, StringUtils.EMPTY); 732 return -1; 733 } 734 735 // handle empty token 736 final int delimLen = getDelimiterMatcher().isMatch(srcChars, start, start, len); 737 if (delimLen > 0) { 738 addToken(tokenList, StringUtils.EMPTY); 739 return start + delimLen; 740 } 741 742 // handle found token 743 final int quoteLen = getQuoteMatcher().isMatch(srcChars, start, start, len); 744 if (quoteLen > 0) { 745 return readWithQuotes(srcChars, start + quoteLen, len, workArea, tokenList, start, quoteLen); 746 } 747 return readWithQuotes(srcChars, start, len, workArea, tokenList, 0, 0); 748 } 749 750 /** 751 * Reads a possibly quoted string token. 752 * 753 * @param srcChars the character array being tokenized 754 * @param start the first character of field 755 * @param len the length of the character array being tokenized 756 * @param workArea a temporary work area 757 * @param tokenList the list of parsed tokens 758 * @param quoteStart the start position of the matched quote, 0 if no quoting 759 * @param quoteLen the length of the matched quote, 0 if no quoting 760 * @return The starting position of the next field (the character 761 * immediately after the delimiter, or if end of string found, 762 * then the length of string 763 */ 764 private int readWithQuotes(final char[] srcChars, final int start, final int len, final StrBuilder workArea, 765 final List<String> tokenList, final int quoteStart, final int quoteLen) { 766 // Loop until we've found the end of the quoted 767 // string or the end of the input 768 workArea.clear(); 769 int pos = start; 770 boolean quoting = quoteLen > 0; 771 int trimStart = 0; 772 773 while (pos < len) { 774 // quoting mode can occur several times throughout a string 775 // we must switch between quoting and non-quoting until we 776 // encounter a non-quoted delimiter, or end of string 777 if (quoting) { 778 // In quoting mode 779 780 // If we've found a quote character, see if it's 781 // followed by a second quote. If so, then we need 782 // to actually put the quote character into the token 783 // rather than end the token. 784 if (isQuote(srcChars, pos, len, quoteStart, quoteLen)) { 785 if (isQuote(srcChars, pos + quoteLen, len, quoteStart, quoteLen)) { 786 // matched pair of quotes, thus an escaped quote 787 workArea.append(srcChars, pos, quoteLen); 788 pos += quoteLen * 2; 789 trimStart = workArea.size(); 790 continue; 791 } 792 793 // end of quoting 794 quoting = false; 795 pos += quoteLen; 796 continue; 797 } 798 799 } else { 800 // Not in quoting mode 801 802 // check for delimiter, and thus end of token 803 final int delimLen = getDelimiterMatcher().isMatch(srcChars, pos, start, len); 804 if (delimLen > 0) { 805 // return condition when end of token found 806 addToken(tokenList, workArea.substring(0, trimStart)); 807 return pos + delimLen; 808 } 809 810 // check for quote, and thus back into quoting mode 811 if (quoteLen > 0 && isQuote(srcChars, pos, len, quoteStart, quoteLen)) { 812 quoting = true; 813 pos += quoteLen; 814 continue; 815 } 816 817 // check for ignored (outside quotes), and ignore 818 final int ignoredLen = getIgnoredMatcher().isMatch(srcChars, pos, start, len); 819 if (ignoredLen > 0) { 820 pos += ignoredLen; 821 continue; 822 } 823 824 // check for trimmed character 825 // don't yet know if its at the end, so copy to workArea 826 // use trimStart to keep track of trim at the end 827 final int trimmedLen = getTrimmerMatcher().isMatch(srcChars, pos, start, len); 828 if (trimmedLen > 0) { 829 workArea.append(srcChars, pos, trimmedLen); 830 pos += trimmedLen; 831 continue; 832 } 833 834 } 835 // copy regular character from inside quotes 836 workArea.append(srcChars[pos++]); 837 trimStart = workArea.size(); 838 } 839 840 // return condition when end of string found 841 addToken(tokenList, workArea.substring(0, trimStart)); 842 return -1; 843 } 844 845 /** 846 * Unsupported ListIterator operation. 847 * 848 * @throws UnsupportedOperationException always 849 */ 850 @Override 851 public void remove() { 852 throw new UnsupportedOperationException("remove() is unsupported"); 853 } 854 855 /** 856 * Resets this tokenizer, forgetting all parsing and iteration already completed. 857 * <p> 858 * This method allows the same tokenizer to be reused for the same String. 859 * 860 * @return this, to enable chaining 861 */ 862 public StrTokenizer reset() { 863 tokenPos = 0; 864 tokens = null; 865 return this; 866 } 867 868 /** 869 * Reset this tokenizer, giving it a new input string to parse. 870 * In this manner you can re-use a tokenizer with the same settings 871 * on multiple input lines. 872 * 873 * @param input the new character array to tokenize, not cloned, null sets no text to parse 874 * @return this, to enable chaining 875 */ 876 public StrTokenizer reset(final char[] input) { 877 reset(); 878 if (input != null) { 879 this.chars = input.clone(); 880 } else { 881 this.chars = null; 882 } 883 return this; 884 } 885 886 /** 887 * Reset this tokenizer, giving it a new input string to parse. 888 * In this manner you can re-use a tokenizer with the same settings 889 * on multiple input lines. 890 * 891 * @param input the new string to tokenize, null sets no text to parse 892 * @return this, to enable chaining 893 */ 894 public StrTokenizer reset(final String input) { 895 reset(); 896 if (input != null) { 897 this.chars = input.toCharArray(); 898 } else { 899 this.chars = null; 900 } 901 return this; 902 } 903 904 /** 905 * Unsupported ListIterator operation. 906 * @param obj this parameter ignored. 907 * @throws UnsupportedOperationException always 908 */ 909 @Override 910 public void set(final String obj) { 911 throw new UnsupportedOperationException("set() is unsupported"); 912 } 913 914 /** 915 * Sets the field delimiter character. 916 * 917 * @param delim the delimiter character to use 918 * @return this, to enable chaining 919 */ 920 public StrTokenizer setDelimiterChar(final char delim) { 921 return setDelimiterMatcher(StrMatcher.charMatcher(delim)); 922 } 923 924 /** 925 * Sets the field delimiter matcher. 926 * <p> 927 * The delimiter is used to separate one token from another. 928 * </p> 929 * 930 * @param delim the delimiter matcher to use 931 * @return this, to enable chaining 932 */ 933 public StrTokenizer setDelimiterMatcher(final StrMatcher delim) { 934 if (delim == null) { 935 this.delimMatcher = StrMatcher.noneMatcher(); 936 } else { 937 this.delimMatcher = delim; 938 } 939 return this; 940 } 941 942 /** 943 * Sets the field delimiter string. 944 * 945 * @param delim the delimiter string to use 946 * @return this, to enable chaining 947 */ 948 public StrTokenizer setDelimiterString(final String delim) { 949 return setDelimiterMatcher(StrMatcher.stringMatcher(delim)); 950 } 951 952 /** 953 * Sets whether the tokenizer should return empty tokens as null. 954 * The default for this property is false. 955 * 956 * @param emptyAsNull whether empty tokens are returned as null 957 * @return this, to enable chaining 958 */ 959 public StrTokenizer setEmptyTokenAsNull(final boolean emptyAsNull) { 960 this.emptyAsNull = emptyAsNull; 961 return this; 962 } 963 964 /** 965 * Sets the character to ignore. 966 * <p> 967 * This character is ignored when parsing the String, unless it is 968 * within a quoted region. 969 * </p> 970 * 971 * @param ignored the ignored character to use 972 * @return this, to enable chaining 973 */ 974 public StrTokenizer setIgnoredChar(final char ignored) { 975 return setIgnoredMatcher(StrMatcher.charMatcher(ignored)); 976 } 977 978 /** 979 * Sets the matcher for characters to ignore. 980 * <p> 981 * These characters are ignored when parsing the String, unless they are 982 * within a quoted region. 983 * </p> 984 * 985 * @param ignored the ignored matcher to use, null ignored 986 * @return this, to enable chaining 987 */ 988 public StrTokenizer setIgnoredMatcher(final StrMatcher ignored) { 989 if (ignored != null) { 990 this.ignoredMatcher = ignored; 991 } 992 return this; 993 } 994 995 /** 996 * Sets whether the tokenizer should ignore and not return empty tokens. 997 * The default for this property is true. 998 * 999 * @param ignoreEmptyTokens whether empty tokens are not returned 1000 * @return this, to enable chaining 1001 */ 1002 public StrTokenizer setIgnoreEmptyTokens(final boolean ignoreEmptyTokens) { 1003 this.ignoreEmptyTokens = ignoreEmptyTokens; 1004 return this; 1005 } 1006 1007 /** 1008 * Sets the quote character to use. 1009 * <p> 1010 * The quote character is used to wrap data between the tokens. 1011 * This enables delimiters to be entered as data. 1012 * </p> 1013 * 1014 * @param quote the quote character to use 1015 * @return this, to enable chaining 1016 */ 1017 public StrTokenizer setQuoteChar(final char quote) { 1018 return setQuoteMatcher(StrMatcher.charMatcher(quote)); 1019 } 1020 1021 /** 1022 * Sets the quote matcher to use. 1023 * <p> 1024 * The quote character is used to wrap data between the tokens. 1025 * This enables delimiters to be entered as data. 1026 * </p> 1027 * 1028 * @param quote the quote matcher to use, null ignored 1029 * @return this, to enable chaining 1030 */ 1031 public StrTokenizer setQuoteMatcher(final StrMatcher quote) { 1032 if (quote != null) { 1033 this.quoteMatcher = quote; 1034 } 1035 return this; 1036 } 1037 1038 /** 1039 * Sets the matcher for characters to trim. 1040 * <p> 1041 * These characters are trimmed off on each side of the delimiter 1042 * until the token or quote is found. 1043 * </p> 1044 * 1045 * @param trimmer the trimmer matcher to use, null ignored 1046 * @return this, to enable chaining 1047 */ 1048 public StrTokenizer setTrimmerMatcher(final StrMatcher trimmer) { 1049 if (trimmer != null) { 1050 this.trimmerMatcher = trimmer; 1051 } 1052 return this; 1053 } 1054 1055 /** 1056 * Gets the number of tokens found in the String. 1057 * 1058 * @return The number of matched tokens 1059 */ 1060 public int size() { 1061 checkTokenized(); 1062 return tokens.length; 1063 } 1064 1065 /** 1066 * Internal method to performs the tokenization. 1067 * <p> 1068 * Most users of this class do not need to call this method. This method 1069 * will be called automatically by other (public) methods when required. 1070 * </p> 1071 * <p> 1072 * This method exists to allow subclasses to add code before or after the 1073 * tokenization. For example, a subclass could alter the character array, 1074 * offset or count to be parsed, or call the tokenizer multiple times on 1075 * multiple strings. It is also be possible to filter the results. 1076 * </p> 1077 * <p> 1078 * {@code StrTokenizer} will always pass a zero offset and a count 1079 * equal to the length of the array to this method, however a subclass 1080 * may pass other values, or even an entirely different array. 1081 * </p> 1082 * 1083 * @param srcChars the character array being tokenized, may be null 1084 * @param offset the start position within the character array, must be valid 1085 * @param count the number of characters to tokenize, must be valid 1086 * @return The modifiable list of String tokens, unmodifiable if null array or zero count 1087 */ 1088 protected List<String> tokenize(final char[] srcChars, final int offset, final int count) { 1089 if (srcChars == null || count == 0) { 1090 return Collections.emptyList(); 1091 } 1092 final StrBuilder buf = new StrBuilder(); 1093 final List<String> tokenList = new ArrayList<>(); 1094 int pos = offset; 1095 1096 // loop around the entire buffer 1097 while (pos >= 0 && pos < count) { 1098 // find next token 1099 pos = readNextToken(srcChars, pos, count, buf, tokenList); 1100 1101 // handle case where end of string is a delimiter 1102 if (pos >= count) { 1103 addToken(tokenList, StringUtils.EMPTY); 1104 } 1105 } 1106 return tokenList; 1107 } 1108 1109 /** 1110 * Gets the String content that the tokenizer is parsing. 1111 * 1112 * @return The string content being parsed 1113 */ 1114 @Override 1115 public String toString() { 1116 if (tokens == null) { 1117 return "StrTokenizer[not tokenized yet]"; 1118 } 1119 return "StrTokenizer" + getTokenList(); 1120 } 1121 1122 }