StrTokenizer.java
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.commons.text;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.ListIterator;
import java.util.NoSuchElementException;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
/**
* Tokenizes a string based on delimiters (separators)
* and supporting quoting and ignored character concepts.
* <p>
* This class can split a String into many smaller strings. It aims
* to do a similar job to {@link java.util.StringTokenizer StringTokenizer},
* however it offers much more control and flexibility including implementing
* the {@code ListIterator} interface. By default, it is set up
* like {@code StringTokenizer}.
* <p>
* The input String is split into a number of <i>tokens</i>.
* Each token is separated from the next String by a <i>delimiter</i>.
* One or more delimiter characters must be specified.
* <p>
* Each token may be surrounded by quotes.
* The <i>quote</i> matcher specifies the quote character(s).
* A quote may be escaped within a quoted section by duplicating itself.
* <p>
* Between each token and the delimiter are potentially characters that need trimming.
* The <i>trimmer</i> matcher specifies these characters.
* One usage might be to trim whitespace characters.
* <p>
* At any point outside the quotes there might potentially be invalid characters.
* The <i>ignored</i> matcher specifies these characters to be removed.
* One usage might be to remove new line characters.
* <p>
* Empty tokens may be removed or returned as null.
* <pre>
* "a,b,c" - Three tokens "a","b","c" (comma delimiter)
* " a, b , c " - Three tokens "a","b","c" (default CSV processing trims whitespace)
* "a, ", b ,", c" - Three tokens "a, " , " b ", ", c" (quoted text untouched)
* </pre>
*
* <table>
* <caption>StrTokenizer properties and options</caption>
* <tr>
* <th>Property</th><th>Type</th><th>Default</th>
* </tr>
* <tr>
* <td>delim</td><td>CharSetMatcher</td><td>{ \t\n\r\f}</td>
* </tr>
* <tr>
* <td>quote</td><td>NoneMatcher</td><td>{}</td>
* </tr>
* <tr>
* <td>ignore</td><td>NoneMatcher</td><td>{}</td>
* </tr>
* <tr>
* <td>emptyTokenAsNull</td><td>boolean</td><td>false</td>
* </tr>
* <tr>
* <td>ignoreEmptyTokens</td><td>boolean</td><td>true</td>
* </tr>
* </table>
*
* @since 1.0
* @deprecated Deprecated as of 1.3, use {@link StringTokenizer} instead. This class will be removed in 2.0.
*/
@Deprecated
public class StrTokenizer implements ListIterator<String>, Cloneable {
/** Comma separated values tokenizer internal variable. */
private static final StrTokenizer CSV_TOKENIZER_PROTOTYPE;
/** Tab separated values tokenizer internal variable. */
private static final StrTokenizer TSV_TOKENIZER_PROTOTYPE;
static {
CSV_TOKENIZER_PROTOTYPE = new StrTokenizer();
CSV_TOKENIZER_PROTOTYPE.setDelimiterMatcher(StrMatcher.commaMatcher());
CSV_TOKENIZER_PROTOTYPE.setQuoteMatcher(StrMatcher.doubleQuoteMatcher());
CSV_TOKENIZER_PROTOTYPE.setIgnoredMatcher(StrMatcher.noneMatcher());
CSV_TOKENIZER_PROTOTYPE.setTrimmerMatcher(StrMatcher.trimMatcher());
CSV_TOKENIZER_PROTOTYPE.setEmptyTokenAsNull(false);
CSV_TOKENIZER_PROTOTYPE.setIgnoreEmptyTokens(false);
TSV_TOKENIZER_PROTOTYPE = new StrTokenizer();
TSV_TOKENIZER_PROTOTYPE.setDelimiterMatcher(StrMatcher.tabMatcher());
TSV_TOKENIZER_PROTOTYPE.setQuoteMatcher(StrMatcher.doubleQuoteMatcher());
TSV_TOKENIZER_PROTOTYPE.setIgnoredMatcher(StrMatcher.noneMatcher());
TSV_TOKENIZER_PROTOTYPE.setTrimmerMatcher(StrMatcher.trimMatcher());
TSV_TOKENIZER_PROTOTYPE.setEmptyTokenAsNull(false);
TSV_TOKENIZER_PROTOTYPE.setIgnoreEmptyTokens(false);
}
/**
* Returns a clone of {@code CSV_TOKENIZER_PROTOTYPE}.
*
* @return a clone of {@code CSV_TOKENIZER_PROTOTYPE}.
*/
private static StrTokenizer getCSVClone() {
return (StrTokenizer) CSV_TOKENIZER_PROTOTYPE.clone();
}
/**
* Gets a new tokenizer instance which parses Comma Separated Value strings
* initializing it with the given input. The default for CSV processing
* will be trim whitespace from both ends (which can be overridden with
* the setTrimmer method).
* <p>
* You must call a "reset" method to set the string which you want to parse.
* </p>
* @return a new tokenizer instance which parses Comma Separated Value strings
*/
public static StrTokenizer getCSVInstance() {
return getCSVClone();
}
/**
* Gets a new tokenizer instance which parses Comma Separated Value strings
* initializing it with the given input. The default for CSV processing
* will be trim whitespace from both ends (which can be overridden with
* the setTrimmer method).
*
* @param input the text to parse
* @return a new tokenizer instance which parses Comma Separated Value strings
*/
public static StrTokenizer getCSVInstance(final char[] input) {
final StrTokenizer tok = getCSVClone();
tok.reset(input);
return tok;
}
/**
* Gets a new tokenizer instance which parses Comma Separated Value strings
* initializing it with the given input. The default for CSV processing
* will be trim whitespace from both ends (which can be overridden with
* the setTrimmer method).
*
* @param input the text to parse
* @return a new tokenizer instance which parses Comma Separated Value strings
*/
public static StrTokenizer getCSVInstance(final String input) {
final StrTokenizer tok = getCSVClone();
tok.reset(input);
return tok;
}
/**
* Returns a clone of {@code TSV_TOKENIZER_PROTOTYPE}.
*
* @return a clone of {@code TSV_TOKENIZER_PROTOTYPE}.
*/
private static StrTokenizer getTSVClone() {
return (StrTokenizer) TSV_TOKENIZER_PROTOTYPE.clone();
}
/**
* Gets a new tokenizer instance which parses Tab Separated Value strings.
* The default for CSV processing will be trim whitespace from both ends
* (which can be overridden with the setTrimmer method).
* <p>
* You must call a "reset" method to set the string which you want to parse.
* </p>
* @return a new tokenizer instance which parses Tab Separated Value strings.
*/
public static StrTokenizer getTSVInstance() {
return getTSVClone();
}
/**
* Gets a new tokenizer instance which parses Tab Separated Value strings.
* The default for CSV processing will be trim whitespace from both ends
* (which can be overridden with the setTrimmer method).
* @param input the string to parse
* @return a new tokenizer instance which parses Tab Separated Value strings.
*/
public static StrTokenizer getTSVInstance(final char[] input) {
final StrTokenizer tok = getTSVClone();
tok.reset(input);
return tok;
}
/**
* Gets a new tokenizer instance which parses Tab Separated Value strings.
* The default for CSV processing will be trim whitespace from both ends
* (which can be overridden with the setTrimmer method).
* @param input the string to parse
* @return a new tokenizer instance which parses Tab Separated Value strings.
*/
public static StrTokenizer getTSVInstance(final String input) {
final StrTokenizer tok = getTSVClone();
tok.reset(input);
return tok;
}
/** The text to work on. */
private char[] chars;
/** The parsed tokens. */
private String[] tokens;
/** The current iteration position. */
private int tokenPos;
/** The delimiter matcher. */
private StrMatcher delimMatcher = StrMatcher.splitMatcher();
/** The quote matcher. */
private StrMatcher quoteMatcher = StrMatcher.noneMatcher();
/** The ignored matcher. */
private StrMatcher ignoredMatcher = StrMatcher.noneMatcher();
/** The trimmer matcher. */
private StrMatcher trimmerMatcher = StrMatcher.noneMatcher();
/** Whether to return empty tokens as null. */
private boolean emptyAsNull;
/** Whether to ignore empty tokens. */
private boolean ignoreEmptyTokens = true;
/**
* Constructs a tokenizer splitting on space, tab, newline and form feed
* as per StringTokenizer, but with no text to tokenize.
* <p>
* This constructor is normally used with {@link #reset(String)}.
* </p>
*/
public StrTokenizer() {
this.chars = null;
}
/**
* Constructs a tokenizer splitting on space, tab, newline and form feed
* as per StringTokenizer.
*
* @param input the string which is to be parsed, not cloned
*/
public StrTokenizer(final char[] input) {
if (input == null) {
this.chars = null;
} else {
this.chars = input.clone();
}
}
/**
* Constructs a tokenizer splitting on the specified character.
*
* @param input the string which is to be parsed, not cloned
* @param delim the field delimiter character
*/
public StrTokenizer(final char[] input, final char delim) {
this(input);
setDelimiterChar(delim);
}
/**
* Constructs a tokenizer splitting on the specified delimiter character
* and handling quotes using the specified quote character.
*
* @param input the string which is to be parsed, not cloned
* @param delim the field delimiter character
* @param quote the field quoted string character
*/
public StrTokenizer(final char[] input, final char delim, final char quote) {
this(input, delim);
setQuoteChar(quote);
}
/**
* Constructs a tokenizer splitting on the specified string.
*
* @param input the string which is to be parsed, not cloned
* @param delim the field delimiter string
*/
public StrTokenizer(final char[] input, final String delim) {
this(input);
setDelimiterString(delim);
}
/**
* Constructs a tokenizer splitting using the specified delimiter matcher.
*
* @param input the string which is to be parsed, not cloned
* @param delim the field delimiter matcher
*/
public StrTokenizer(final char[] input, final StrMatcher delim) {
this(input);
setDelimiterMatcher(delim);
}
/**
* Constructs a tokenizer splitting using the specified delimiter matcher
* and handling quotes using the specified quote matcher.
*
* @param input the string which is to be parsed, not cloned
* @param delim the field delimiter character
* @param quote the field quoted string character
*/
public StrTokenizer(final char[] input, final StrMatcher delim, final StrMatcher quote) {
this(input, delim);
setQuoteMatcher(quote);
}
/**
* Constructs a tokenizer splitting on space, tab, newline and form feed
* as per StringTokenizer.
*
* @param input the string which is to be parsed
*/
public StrTokenizer(final String input) {
if (input != null) {
chars = input.toCharArray();
} else {
chars = null;
}
}
/**
* Constructs a tokenizer splitting on the specified delimiter character.
*
* @param input the string which is to be parsed
* @param delim the field delimiter character
*/
public StrTokenizer(final String input, final char delim) {
this(input);
setDelimiterChar(delim);
}
/**
* Constructs a tokenizer splitting on the specified delimiter character
* and handling quotes using the specified quote character.
*
* @param input the string which is to be parsed
* @param delim the field delimiter character
* @param quote the field quoted string character
*/
public StrTokenizer(final String input, final char delim, final char quote) {
this(input, delim);
setQuoteChar(quote);
}
/**
* Constructs a tokenizer splitting on the specified delimiter string.
*
* @param input the string which is to be parsed
* @param delim the field delimiter string
*/
public StrTokenizer(final String input, final String delim) {
this(input);
setDelimiterString(delim);
}
/**
* Constructs a tokenizer splitting using the specified delimiter matcher.
*
* @param input the string which is to be parsed
* @param delim the field delimiter matcher
*/
public StrTokenizer(final String input, final StrMatcher delim) {
this(input);
setDelimiterMatcher(delim);
}
/**
* Constructs a tokenizer splitting using the specified delimiter matcher
* and handling quotes using the specified quote matcher.
*
* @param input the string which is to be parsed
* @param delim the field delimiter matcher
* @param quote the field quoted string matcher
*/
public StrTokenizer(final String input, final StrMatcher delim, final StrMatcher quote) {
this(input, delim);
setQuoteMatcher(quote);
}
/**
* Unsupported ListIterator operation.
* @param obj this parameter ignored.
* @throws UnsupportedOperationException always
*/
@Override
public void add(final String obj) {
throw new UnsupportedOperationException("add() is unsupported");
}
/**
* Adds a token to a list, paying attention to the parameters we've set.
*
* @param list the list to add to
* @param tok the token to add
*/
private void addToken(final List<String> list, String tok) {
if (tok == null || tok.isEmpty()) {
if (isIgnoreEmptyTokens()) {
return;
}
if (isEmptyTokenAsNull()) {
tok = null;
}
}
list.add(tok);
}
/**
* Checks if tokenization has been done, and if not then do it.
*/
private void checkTokenized() {
if (tokens == null) {
if (chars == null) {
// still call tokenize as subclass may do some work
final List<String> split = tokenize(null, 0, 0);
tokens = split.toArray(ArrayUtils.EMPTY_STRING_ARRAY);
} else {
final List<String> split = tokenize(chars, 0, chars.length);
tokens = split.toArray(ArrayUtils.EMPTY_STRING_ARRAY);
}
}
}
/**
* Creates a new instance of this Tokenizer. The new instance is reset so
* that it will be at the start of the token list.
* If a {@link CloneNotSupportedException} is caught, return {@code null}.
*
* @return a new instance of this Tokenizer which has been reset.
*/
@Override
public Object clone() {
try {
return cloneReset();
} catch (final CloneNotSupportedException ex) {
return null;
}
}
/**
* Creates a new instance of this Tokenizer. The new instance is reset so that
* it will be at the start of the token list.
*
* @return a new instance of this Tokenizer which has been reset.
* @throws CloneNotSupportedException if there is a problem cloning
*/
Object cloneReset() throws CloneNotSupportedException {
// this method exists to enable 100% test coverage
final StrTokenizer cloned = (StrTokenizer) super.clone();
if (cloned.chars != null) {
cloned.chars = cloned.chars.clone();
}
cloned.reset();
return cloned;
}
/**
* Gets the String content that the tokenizer is parsing.
*
* @return The string content being parsed
*/
public String getContent() {
if (chars == null) {
return null;
}
return new String(chars);
}
/**
* Gets the field delimiter matcher.
*
* @return The delimiter matcher in use
*/
public StrMatcher getDelimiterMatcher() {
return this.delimMatcher;
}
/**
* Gets the ignored character matcher.
* <p>
* These characters are ignored when parsing the String, unless they are
* within a quoted region.
* The default value is not to ignore anything.
* </p>
*
* @return The ignored matcher in use
*/
public StrMatcher getIgnoredMatcher() {
return ignoredMatcher;
}
/**
* Gets the quote matcher currently in use.
* <p>
* The quote character is used to wrap data between the tokens.
* This enables delimiters to be entered as data.
* The default value is '"' (double quote).
* </p>
*
* @return The quote matcher in use
*/
public StrMatcher getQuoteMatcher() {
return quoteMatcher;
}
/**
* Gets a copy of the full token list as an independent modifiable array.
*
* @return The tokens as a String array
*/
public String[] getTokenArray() {
checkTokenized();
return tokens.clone();
}
/**
* Gets a copy of the full token list as an independent modifiable list.
*
* @return The tokens as a String array
*/
public List<String> getTokenList() {
checkTokenized();
final List<String> list = new ArrayList<>(tokens.length);
Collections.addAll(list, tokens);
return list;
}
/**
* Gets the trimmer character matcher.
* <p>
* These characters are trimmed off on each side of the delimiter
* until the token or quote is found.
* The default value is not to trim anything.
* </p>
*
* @return The trimmer matcher in use
*/
public StrMatcher getTrimmerMatcher() {
return trimmerMatcher;
}
/**
* Checks whether there are any more tokens.
*
* @return true if there are more tokens
*/
@Override
public boolean hasNext() {
checkTokenized();
return tokenPos < tokens.length;
}
/**
* Checks whether there are any previous tokens that can be iterated to.
*
* @return true if there are previous tokens
*/
@Override
public boolean hasPrevious() {
checkTokenized();
return tokenPos > 0;
}
/**
* Gets whether the tokenizer currently returns empty tokens as null.
* The default for this property is false.
*
* @return true if empty tokens are returned as null
*/
public boolean isEmptyTokenAsNull() {
return this.emptyAsNull;
}
/**
* Gets whether the tokenizer currently ignores empty tokens.
* The default for this property is true.
*
* @return true if empty tokens are not returned
*/
public boolean isIgnoreEmptyTokens() {
return ignoreEmptyTokens;
}
/**
* Checks if the characters at the index specified match the quote
* already matched in readNextToken().
*
* @param srcChars the character array being tokenized
* @param pos the position to check for a quote
* @param len the length of the character array being tokenized
* @param quoteStart the start position of the matched quote, 0 if no quoting
* @param quoteLen the length of the matched quote, 0 if no quoting
* @return true if a quote is matched
*/
private boolean isQuote(final char[] srcChars,
final int pos,
final int len,
final int quoteStart,
final int quoteLen) {
for (int i = 0; i < quoteLen; i++) {
if (pos + i >= len || srcChars[pos + i] != srcChars[quoteStart + i]) {
return false;
}
}
return true;
}
/**
* Gets the next token.
*
* @return The next String token
* @throws NoSuchElementException if there are no more elements
*/
@Override
public String next() {
if (hasNext()) {
return tokens[tokenPos++];
}
throw new NoSuchElementException();
}
/**
* Gets the index of the next token to return.
*
* @return The next token index
*/
@Override
public int nextIndex() {
return tokenPos;
}
/**
* Gets the next token from the String.
* Equivalent to {@link #next()} except it returns null rather than
* throwing {@link NoSuchElementException} when no tokens remain.
*
* @return The next sequential token, or null when no more tokens are found
*/
public String nextToken() {
if (hasNext()) {
return tokens[tokenPos++];
}
return null;
}
/**
* Gets the token previous to the last returned token.
*
* @return The previous token
*/
@Override
public String previous() {
if (hasPrevious()) {
return tokens[--tokenPos];
}
throw new NoSuchElementException();
}
/**
* Gets the index of the previous token.
*
* @return The previous token index
*/
@Override
public int previousIndex() {
return tokenPos - 1;
}
/**
* Gets the previous token from the String.
*
* @return The previous sequential token, or null when no more tokens are found
*/
public String previousToken() {
if (hasPrevious()) {
return tokens[--tokenPos];
}
return null;
}
/**
* Reads character by character through the String to get the next token.
*
* @param srcChars the character array being tokenized
* @param start the first character of field
* @param len the length of the character array being tokenized
* @param workArea a temporary work area
* @param tokenList the list of parsed tokens
* @return The starting position of the next field (the character
* immediately after the delimiter), or -1 if end of string found
*/
private int readNextToken(final char[] srcChars,
int start,
final int len,
final StrBuilder workArea,
final List<String> tokenList) {
// skip all leading whitespace, unless it is the
// field delimiter or the quote character
while (start < len) {
final int removeLen = Math.max(
getIgnoredMatcher().isMatch(srcChars, start, start, len),
getTrimmerMatcher().isMatch(srcChars, start, start, len));
if (removeLen == 0
|| getDelimiterMatcher().isMatch(srcChars, start, start, len) > 0
|| getQuoteMatcher().isMatch(srcChars, start, start, len) > 0) {
break;
}
start += removeLen;
}
// handle reaching end
if (start >= len) {
addToken(tokenList, StringUtils.EMPTY);
return -1;
}
// handle empty token
final int delimLen = getDelimiterMatcher().isMatch(srcChars, start, start, len);
if (delimLen > 0) {
addToken(tokenList, StringUtils.EMPTY);
return start + delimLen;
}
// handle found token
final int quoteLen = getQuoteMatcher().isMatch(srcChars, start, start, len);
if (quoteLen > 0) {
return readWithQuotes(srcChars, start + quoteLen, len, workArea, tokenList, start, quoteLen);
}
return readWithQuotes(srcChars, start, len, workArea, tokenList, 0, 0);
}
/**
* Reads a possibly quoted string token.
*
* @param srcChars the character array being tokenized
* @param start the first character of field
* @param len the length of the character array being tokenized
* @param workArea a temporary work area
* @param tokenList the list of parsed tokens
* @param quoteStart the start position of the matched quote, 0 if no quoting
* @param quoteLen the length of the matched quote, 0 if no quoting
* @return The starting position of the next field (the character
* immediately after the delimiter, or if end of string found,
* then the length of string
*/
private int readWithQuotes(final char[] srcChars, final int start, final int len, final StrBuilder workArea,
final List<String> tokenList, final int quoteStart, final int quoteLen) {
// Loop until we've found the end of the quoted
// string or the end of the input
workArea.clear();
int pos = start;
boolean quoting = quoteLen > 0;
int trimStart = 0;
while (pos < len) {
// quoting mode can occur several times throughout a string
// we must switch between quoting and non-quoting until we
// encounter a non-quoted delimiter, or end of string
if (quoting) {
// In quoting mode
// If we've found a quote character, see if it's
// followed by a second quote. If so, then we need
// to actually put the quote character into the token
// rather than end the token.
if (isQuote(srcChars, pos, len, quoteStart, quoteLen)) {
if (isQuote(srcChars, pos + quoteLen, len, quoteStart, quoteLen)) {
// matched pair of quotes, thus an escaped quote
workArea.append(srcChars, pos, quoteLen);
pos += quoteLen * 2;
trimStart = workArea.size();
continue;
}
// end of quoting
quoting = false;
pos += quoteLen;
continue;
}
} else {
// Not in quoting mode
// check for delimiter, and thus end of token
final int delimLen = getDelimiterMatcher().isMatch(srcChars, pos, start, len);
if (delimLen > 0) {
// return condition when end of token found
addToken(tokenList, workArea.substring(0, trimStart));
return pos + delimLen;
}
// check for quote, and thus back into quoting mode
if (quoteLen > 0 && isQuote(srcChars, pos, len, quoteStart, quoteLen)) {
quoting = true;
pos += quoteLen;
continue;
}
// check for ignored (outside quotes), and ignore
final int ignoredLen = getIgnoredMatcher().isMatch(srcChars, pos, start, len);
if (ignoredLen > 0) {
pos += ignoredLen;
continue;
}
// check for trimmed character
// don't yet know if its at the end, so copy to workArea
// use trimStart to keep track of trim at the end
final int trimmedLen = getTrimmerMatcher().isMatch(srcChars, pos, start, len);
if (trimmedLen > 0) {
workArea.append(srcChars, pos, trimmedLen);
pos += trimmedLen;
continue;
}
}
// copy regular character from inside quotes
workArea.append(srcChars[pos++]);
trimStart = workArea.size();
}
// return condition when end of string found
addToken(tokenList, workArea.substring(0, trimStart));
return -1;
}
/**
* Unsupported ListIterator operation.
*
* @throws UnsupportedOperationException always
*/
@Override
public void remove() {
throw new UnsupportedOperationException("remove() is unsupported");
}
/**
* Resets this tokenizer, forgetting all parsing and iteration already completed.
* <p>
* This method allows the same tokenizer to be reused for the same String.
*
* @return this, to enable chaining
*/
public StrTokenizer reset() {
tokenPos = 0;
tokens = null;
return this;
}
/**
* Reset this tokenizer, giving it a new input string to parse.
* In this manner you can re-use a tokenizer with the same settings
* on multiple input lines.
*
* @param input the new character array to tokenize, not cloned, null sets no text to parse
* @return this, to enable chaining
*/
public StrTokenizer reset(final char[] input) {
reset();
if (input != null) {
this.chars = input.clone();
} else {
this.chars = null;
}
return this;
}
/**
* Reset this tokenizer, giving it a new input string to parse.
* In this manner you can re-use a tokenizer with the same settings
* on multiple input lines.
*
* @param input the new string to tokenize, null sets no text to parse
* @return this, to enable chaining
*/
public StrTokenizer reset(final String input) {
reset();
if (input != null) {
this.chars = input.toCharArray();
} else {
this.chars = null;
}
return this;
}
/**
* Unsupported ListIterator operation.
* @param obj this parameter ignored.
* @throws UnsupportedOperationException always
*/
@Override
public void set(final String obj) {
throw new UnsupportedOperationException("set() is unsupported");
}
/**
* Sets the field delimiter character.
*
* @param delim the delimiter character to use
* @return this, to enable chaining
*/
public StrTokenizer setDelimiterChar(final char delim) {
return setDelimiterMatcher(StrMatcher.charMatcher(delim));
}
/**
* Sets the field delimiter matcher.
* <p>
* The delimiter is used to separate one token from another.
* </p>
*
* @param delim the delimiter matcher to use
* @return this, to enable chaining
*/
public StrTokenizer setDelimiterMatcher(final StrMatcher delim) {
if (delim == null) {
this.delimMatcher = StrMatcher.noneMatcher();
} else {
this.delimMatcher = delim;
}
return this;
}
/**
* Sets the field delimiter string.
*
* @param delim the delimiter string to use
* @return this, to enable chaining
*/
public StrTokenizer setDelimiterString(final String delim) {
return setDelimiterMatcher(StrMatcher.stringMatcher(delim));
}
/**
* Sets whether the tokenizer should return empty tokens as null.
* The default for this property is false.
*
* @param emptyAsNull whether empty tokens are returned as null
* @return this, to enable chaining
*/
public StrTokenizer setEmptyTokenAsNull(final boolean emptyAsNull) {
this.emptyAsNull = emptyAsNull;
return this;
}
/**
* Sets the character to ignore.
* <p>
* This character is ignored when parsing the String, unless it is
* within a quoted region.
* </p>
*
* @param ignored the ignored character to use
* @return this, to enable chaining
*/
public StrTokenizer setIgnoredChar(final char ignored) {
return setIgnoredMatcher(StrMatcher.charMatcher(ignored));
}
/**
* Sets the matcher for characters to ignore.
* <p>
* These characters are ignored when parsing the String, unless they are
* within a quoted region.
* </p>
*
* @param ignored the ignored matcher to use, null ignored
* @return this, to enable chaining
*/
public StrTokenizer setIgnoredMatcher(final StrMatcher ignored) {
if (ignored != null) {
this.ignoredMatcher = ignored;
}
return this;
}
/**
* Sets whether the tokenizer should ignore and not return empty tokens.
* The default for this property is true.
*
* @param ignoreEmptyTokens whether empty tokens are not returned
* @return this, to enable chaining
*/
public StrTokenizer setIgnoreEmptyTokens(final boolean ignoreEmptyTokens) {
this.ignoreEmptyTokens = ignoreEmptyTokens;
return this;
}
/**
* Sets the quote character to use.
* <p>
* The quote character is used to wrap data between the tokens.
* This enables delimiters to be entered as data.
* </p>
*
* @param quote the quote character to use
* @return this, to enable chaining
*/
public StrTokenizer setQuoteChar(final char quote) {
return setQuoteMatcher(StrMatcher.charMatcher(quote));
}
/**
* Sets the quote matcher to use.
* <p>
* The quote character is used to wrap data between the tokens.
* This enables delimiters to be entered as data.
* </p>
*
* @param quote the quote matcher to use, null ignored
* @return this, to enable chaining
*/
public StrTokenizer setQuoteMatcher(final StrMatcher quote) {
if (quote != null) {
this.quoteMatcher = quote;
}
return this;
}
/**
* Sets the matcher for characters to trim.
* <p>
* These characters are trimmed off on each side of the delimiter
* until the token or quote is found.
* </p>
*
* @param trimmer the trimmer matcher to use, null ignored
* @return this, to enable chaining
*/
public StrTokenizer setTrimmerMatcher(final StrMatcher trimmer) {
if (trimmer != null) {
this.trimmerMatcher = trimmer;
}
return this;
}
/**
* Gets the number of tokens found in the String.
*
* @return The number of matched tokens
*/
public int size() {
checkTokenized();
return tokens.length;
}
/**
* Internal method to performs the tokenization.
* <p>
* Most users of this class do not need to call this method. This method
* will be called automatically by other (public) methods when required.
* </p>
* <p>
* This method exists to allow subclasses to add code before or after the
* tokenization. For example, a subclass could alter the character array,
* offset or count to be parsed, or call the tokenizer multiple times on
* multiple strings. It is also be possible to filter the results.
* </p>
* <p>
* {@code StrTokenizer} will always pass a zero offset and a count
* equal to the length of the array to this method, however a subclass
* may pass other values, or even an entirely different array.
* </p>
*
* @param srcChars the character array being tokenized, may be null
* @param offset the start position within the character array, must be valid
* @param count the number of characters to tokenize, must be valid
* @return The modifiable list of String tokens, unmodifiable if null array or zero count
*/
protected List<String> tokenize(final char[] srcChars, final int offset, final int count) {
if (srcChars == null || count == 0) {
return Collections.emptyList();
}
final StrBuilder buf = new StrBuilder();
final List<String> tokenList = new ArrayList<>();
int pos = offset;
// loop around the entire buffer
while (pos >= 0 && pos < count) {
// find next token
pos = readNextToken(srcChars, pos, count, buf, tokenList);
// handle case where end of string is a delimiter
if (pos >= count) {
addToken(tokenList, StringUtils.EMPTY);
}
}
return tokenList;
}
/**
* Gets the String content that the tokenizer is parsing.
*
* @return The string content being parsed
*/
@Override
public String toString() {
if (tokens == null) {
return "StrTokenizer[not tokenized yet]";
}
return "StrTokenizer" + getTokenList();
}
}