001/* 002 * Licensed to the Apache Software Foundation (ASF) under one or more 003 * contributor license agreements. See the NOTICE file distributed with 004 * this work for additional information regarding copyright ownership. 005 * The ASF licenses this file to You under the Apache License, Version 2.0 006 * (the "License"); you may not use this file except in compliance with 007 * the License. You may obtain a copy of the License at 008 * 009 * http://www.apache.org/licenses/LICENSE-2.0 010 * 011 * Unless required by applicable law or agreed to in writing, software 012 * distributed under the License is distributed on an "AS IS" BASIS, 013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 014 * See the License for the specific language governing permissions and 015 * limitations under the License. 016 */ 017package org.apache.commons.io.input; 018 019import java.io.BufferedInputStream; 020import java.io.BufferedReader; 021import java.io.File; 022import java.io.IOException; 023import java.io.InputStream; 024import java.io.InputStreamReader; 025import java.io.Reader; 026import java.io.StringReader; 027import java.net.HttpURLConnection; 028import java.net.URL; 029import java.net.URLConnection; 030import java.nio.charset.Charset; 031import java.nio.charset.StandardCharsets; 032import java.nio.file.Files; 033import java.nio.file.Path; 034import java.text.MessageFormat; 035import java.util.Locale; 036import java.util.Objects; 037import java.util.regex.Matcher; 038import java.util.regex.Pattern; 039 040import org.apache.commons.io.ByteOrderMark; 041import org.apache.commons.io.Charsets; 042import org.apache.commons.io.IOUtils; 043import org.apache.commons.io.build.AbstractStreamBuilder; 044import org.apache.commons.io.function.IOConsumer; 045import org.apache.commons.io.output.XmlStreamWriter; 046 047/** 048 * Character stream that handles all the necessary Voodoo to figure out the charset encoding of the XML document within the stream. 049 * <p> 050 * IMPORTANT: This class is not related in any way to the org.xml.sax.XMLReader. This one IS a character stream. 051 * </p> 052 * <p> 053 * All this has to be done without consuming characters from the stream, if not the XML parser will not recognized the document as a valid XML. This is not 100% 054 * true, but it's close enough (UTF-8 BOM is not handled by all parsers right now, XmlStreamReader handles it and things work in all parsers). 055 * </p> 056 * <p> 057 * The XmlStreamReader class handles the charset encoding of XML documents in Files, raw streams and HTTP streams by offering a wide set of constructors. 058 * </p> 059 * <p> 060 * By default the charset encoding detection is lenient, the constructor with the lenient flag can be used for a script (following HTTP MIME and XML 061 * specifications). All this is nicely explained by Mark Pilgrim in his blog, <a href="https://diveintomark.org/archives/2004/02/13/xml-media-types"> 062 * Determining the character encoding of a feed</a>. 063 * </p> 064 * <p> 065 * To build an instance, use {@link Builder}. 066 * </p> 067 * <p> 068 * Originally developed for <a href="https://rome.dev.java.net">ROME</a> under Apache License 2.0. 069 * </p> 070 * 071 * @see Builder 072 * @see org.apache.commons.io.output.XmlStreamWriter 073 * @since 2.0 074 */ 075public class XmlStreamReader extends Reader { 076 077 // @formatter:off 078 /** 079 * Builds a new {@link XmlStreamWriter}. 080 * 081 * Constructs a Reader using an InputStream and the associated content-type header. This constructor is lenient regarding the encoding detection. 082 * <p> 083 * First it checks if the stream has BOM. If there is not BOM checks the content-type encoding. If there is not content-type encoding checks the XML prolog 084 * encoding. If there is not XML prolog encoding uses the default encoding mandated by the content-type MIME type. 085 * </p> 086 * <p> 087 * If lenient detection is indicated and the detection above fails as per specifications it then attempts the following: 088 * </p> 089 * <p> 090 * If the content type was 'text/html' it replaces it with 'text/xml' and tries the detection again. 091 * </p> 092 * <p> 093 * Else if the XML prolog had a charset encoding that encoding is used. 094 * </p> 095 * <p> 096 * Else if the content type had a charset encoding that encoding is used. 097 * </p> 098 * <p> 099 * Else 'UTF-8' is used. 100 * </p> 101 * <p> 102 * If lenient detection is indicated an XmlStreamReaderException is never thrown. 103 * </p> 104 * <p> 105 * For example: 106 * </p> 107 * 108 * <pre>{@code 109 * XmlStreamReader r = XmlStreamReader.builder() 110 * .setPath(path) 111 * .setCharset(StandardCharsets.UTF_8) 112 * .get(); 113 * } 114 * </pre> 115 * 116 * @see #get() 117 * @since 2.12.0 118 */ 119 // @formatter:on 120 public static class Builder extends AbstractStreamBuilder<XmlStreamReader, Builder> { 121 122 private boolean nullCharset = true; 123 private boolean lenient = true; 124 private String httpContentType; 125 126 /** 127 * Constructs a new builder of {@link XmlStreamReader}. 128 */ 129 public Builder() { 130 // empty 131 } 132 133 /** 134 * Builds a new {@link XmlStreamWriter}. 135 * <p> 136 * You must set an aspect that supports {@link #getInputStream()}, otherwise, this method throws an exception. 137 * </p> 138 * <p> 139 * This builder uses the following aspects: 140 * </p> 141 * <ul> 142 * <li>{@link #getInputStream()}</li> 143 * <li>{@link #getCharset()}</li> 144 * <li>lenient</li> 145 * <li>httpContentType</li> 146 * </ul> 147 * 148 * @return a new instance. 149 * @throws IllegalStateException if the {@code origin} is {@code null}. 150 * @throws UnsupportedOperationException if the origin cannot be converted to an {@link InputStream}. 151 * @throws IOException if an I/O error occurs converting to an {@link InputStream} using {@link #getInputStream()}. 152 * @throws XmlStreamReaderException thrown if the Charset encoding could not be determined according to the specification. 153 * @see #getInputStream() 154 * @see #getUnchecked() 155 */ 156 @Override 157 public XmlStreamReader get() throws IOException { 158 final String defaultEncoding = nullCharset ? null : getCharset().name(); 159 // @formatter:off 160 return httpContentType == null 161 ? new XmlStreamReader(getInputStream(), lenient, defaultEncoding) 162 : new XmlStreamReader(getInputStream(), httpContentType, lenient, defaultEncoding); 163 // @formatter:on 164 } 165 166 @Override 167 public Builder setCharset(final Charset charset) { 168 nullCharset = charset == null; 169 return super.setCharset(charset); 170 } 171 172 @Override 173 public Builder setCharset(final String charset) { 174 nullCharset = charset == null; 175 return super.setCharset(Charsets.toCharset(charset, getCharsetDefault())); 176 } 177 178 /** 179 * Sets the HTTP content type. 180 * 181 * @param httpContentType the HTTP content type. 182 * @return {@code this} instance. 183 */ 184 public Builder setHttpContentType(final String httpContentType) { 185 this.httpContentType = httpContentType; 186 return this; 187 } 188 189 /** 190 * Sets the lenient toggle. 191 * 192 * @param lenient the lenient toggle. 193 * @return {@code this} instance. 194 */ 195 public Builder setLenient(final boolean lenient) { 196 this.lenient = lenient; 197 return this; 198 } 199 200 } 201 202 private static final String UTF_8 = StandardCharsets.UTF_8.name(); 203 204 private static final String US_ASCII = StandardCharsets.US_ASCII.name(); 205 206 private static final String UTF_16BE = StandardCharsets.UTF_16BE.name(); 207 208 private static final String UTF_16LE = StandardCharsets.UTF_16LE.name(); 209 210 private static final String UTF_32BE = "UTF-32BE"; 211 212 private static final String UTF_32LE = "UTF-32LE"; 213 214 private static final String UTF_16 = StandardCharsets.UTF_16.name(); 215 216 private static final String UTF_32 = "UTF-32"; 217 218 private static final String EBCDIC = "CP1047"; 219 220 private static final ByteOrderMark[] BOMS = { ByteOrderMark.UTF_8, ByteOrderMark.UTF_16BE, ByteOrderMark.UTF_16LE, ByteOrderMark.UTF_32BE, 221 ByteOrderMark.UTF_32LE }; 222 223 /** UTF_16LE and UTF_32LE have the same two starting BOM bytes. */ 224 private static final ByteOrderMark[] XML_GUESS_BYTES = { new ByteOrderMark(UTF_8, 0x3C, 0x3F, 0x78, 0x6D), 225 new ByteOrderMark(UTF_16BE, 0x00, 0x3C, 0x00, 0x3F), new ByteOrderMark(UTF_16LE, 0x3C, 0x00, 0x3F, 0x00), 226 new ByteOrderMark(UTF_32BE, 0x00, 0x00, 0x00, 0x3C, 0x00, 0x00, 0x00, 0x3F, 0x00, 0x00, 0x00, 0x78, 0x00, 0x00, 0x00, 0x6D), 227 new ByteOrderMark(UTF_32LE, 0x3C, 0x00, 0x00, 0x00, 0x3F, 0x00, 0x00, 0x00, 0x78, 0x00, 0x00, 0x00, 0x6D, 0x00, 0x00, 0x00), 228 new ByteOrderMark(EBCDIC, 0x4C, 0x6F, 0xA7, 0x94) }; 229 230 private static final Pattern CHARSET_PATTERN = Pattern.compile("charset=[\"']?([.[^; \"']]*)[\"']?"); 231 232 /** 233 * Pattern capturing the encoding of the <a href="https://www.w3.org/TR/REC-xml/#sec-pi">{@code 'xml'} processing instruction</a>. 234 * <p> 235 * See also the <a href="https://www.w3.org/TR/2008/REC-xml-20081126/#NT-EncName">NT-EncName</a> XML specification. 236 * </p> 237 * <p> 238 * Note the documented pattern is: 239 * </p> 240 * <pre> 241 * EncName ::= [A-Za-z] ([A-Za-z0-9._] | '-')* 242 * </pre> 243 * <p> 244 * However this does not match all the aliases that are supported by Java. For example, {@code '437'}, {@code 'ISO_8859-1:1987'} and 245 * {@code 'ebcdic-de-273+euro'}. 246 * </p> 247 */ 248 public static final Pattern ENCODING_PATTERN = Pattern.compile( 249 // @formatter:off 250 "^<\\?xml\\s+" 251 + "(?:version\\s*=\\s*(?:(?:\"1\\.[0-9]+\")|(?:'1.[0-9]+'))\\s+)??" 252 + "encoding\\s*=\\s*" 253 + "((?:\"[A-Za-z0-9][A-Za-z0-9._+:-]*\")" // double-quoted 254 + "|(?:'[A-Za-z0-9][A-Za-z0-9._+:-]*'))", // single-quoted 255 Pattern.MULTILINE); 256 // @formatter:on 257 258 private static final String RAW_EX_1 = "Illegal encoding, BOM [{0}] XML guess [{1}] XML prolog [{2}] encoding mismatch"; 259 260 private static final String RAW_EX_2 = "Illegal encoding, BOM [{0}] XML guess [{1}] XML prolog [{2}] unknown BOM"; 261 262 private static final String HTTP_EX_1 = "Illegal encoding, CT-MIME [{0}] CT-Enc [{1}] BOM [{2}] XML guess [{3}] XML prolog [{4}], BOM must be null"; 263 264 private static final String HTTP_EX_2 = "Illegal encoding, CT-MIME [{0}] CT-Enc [{1}] BOM [{2}] XML guess [{3}] XML prolog [{4}], encoding mismatch"; 265 266 private static final String HTTP_EX_3 = "Illegal encoding, CT-MIME [{0}] CT-Enc [{1}] BOM [{2}] XML guess [{3}] XML prolog [{4}], Illegal MIME"; 267 268 /** 269 * Constructs a new {@link Builder}. 270 * 271 * @return a new {@link Builder}. 272 * @since 2.12.0 273 */ 274 public static Builder builder() { 275 return new Builder(); 276 } 277 278 /** 279 * Gets the charset parameter value, {@code null} if not present, {@code null} if httpContentType is {@code null}. 280 * 281 * @param httpContentType the HTTP content type 282 * @return The content type encoding (upcased) 283 */ 284 static String getContentTypeEncoding(final String httpContentType) { 285 String encoding = null; 286 if (httpContentType != null) { 287 final int i = httpContentType.indexOf(";"); 288 if (i > -1) { 289 final String postMime = httpContentType.substring(i + 1); 290 final Matcher m = CHARSET_PATTERN.matcher(postMime); 291 encoding = m.find() ? m.group(1) : null; 292 encoding = encoding != null ? encoding.toUpperCase(Locale.ROOT) : null; 293 } 294 } 295 return encoding; 296 } 297 298 /** 299 * Gets the MIME type or {@code null} if httpContentType is {@code null}. 300 * 301 * @param httpContentType the HTTP content type 302 * @return The mime content type 303 */ 304 static String getContentTypeMime(final String httpContentType) { 305 String mime = null; 306 if (httpContentType != null) { 307 final int i = httpContentType.indexOf(";"); 308 mime = i >= 0 ? httpContentType.substring(0, i) : httpContentType; 309 mime = mime.trim(); 310 } 311 return mime; 312 } 313 314 /** 315 * Gets the encoding declared in the <?xml encoding=...?>, {@code null} if none. 316 * 317 * @param inputStream InputStream to create the reader from. 318 * @param guessedEnc guessed encoding 319 * @return the encoding declared in the <?xml encoding=...?> 320 * @throws IOException thrown if there is a problem reading the stream. 321 */ 322 private static String getXmlProlog(final InputStream inputStream, final String guessedEnc) throws IOException { 323 String encoding = null; 324 if (guessedEnc != null) { 325 final byte[] bytes = IOUtils.byteArray(); 326 inputStream.mark(IOUtils.DEFAULT_BUFFER_SIZE); 327 int offset = 0; 328 int max = IOUtils.DEFAULT_BUFFER_SIZE; 329 int c = inputStream.read(bytes, offset, max); 330 int firstGT = -1; 331 String xmlProlog = ""; // avoid possible NPE warning (cannot happen; this just silences the warning) 332 while (c != -1 && firstGT == -1 && offset < IOUtils.DEFAULT_BUFFER_SIZE) { 333 offset += c; 334 max -= c; 335 c = inputStream.read(bytes, offset, max); 336 xmlProlog = new String(bytes, 0, offset, guessedEnc); 337 firstGT = xmlProlog.indexOf('>'); 338 } 339 if (firstGT == -1) { 340 if (c == -1) { 341 throw new IOException("Unexpected end of XML stream"); 342 } 343 throw new IOException("XML prolog or ROOT element not found on first " + offset + " bytes"); 344 } 345 final int bytesRead = offset; 346 if (bytesRead > 0) { 347 inputStream.reset(); 348 final BufferedReader bReader = new BufferedReader(new StringReader(xmlProlog.substring(0, firstGT + 1))); 349 final StringBuilder prolog = new StringBuilder(); 350 IOConsumer.forEach(bReader.lines(), l -> prolog.append(l).append(' ')); 351 final Matcher m = ENCODING_PATTERN.matcher(prolog); 352 if (m.find()) { 353 encoding = m.group(1).toUpperCase(Locale.ROOT); 354 encoding = encoding.substring(1, encoding.length() - 1); 355 } 356 } 357 } 358 return encoding; 359 } 360 361 /** 362 * Tests if the MIME type belongs to the APPLICATION XML family. 363 * 364 * @param mime The mime type 365 * @return true if the mime type belongs to the APPLICATION XML family, otherwise false 366 */ 367 static boolean isAppXml(final String mime) { 368 return mime != null && (mime.equals("application/xml") || mime.equals("application/xml-dtd") || mime.equals("application/xml-external-parsed-entity") 369 || mime.startsWith("application/") && mime.endsWith("+xml")); 370 } 371 372 /** 373 * Tests if the MIME type belongs to the TEXT XML family. 374 * 375 * @param mime The mime type 376 * @return true if the mime type belongs to the TEXT XML family, otherwise false 377 */ 378 static boolean isTextXml(final String mime) { 379 return mime != null && (mime.equals("text/xml") || mime.equals("text/xml-external-parsed-entity") || mime.startsWith("text/") && mime.endsWith("+xml")); 380 } 381 382 private final Reader reader; 383 384 private final String encoding; 385 386 private final String defaultEncoding; 387 388 /** 389 * Constructs a Reader for a File. 390 * <p> 391 * It looks for the UTF-8 BOM first, if none sniffs the XML prolog charset, if this is also missing defaults to UTF-8. 392 * </p> 393 * <p> 394 * It does a lenient charset encoding detection, check the constructor with the lenient parameter for details. 395 * </p> 396 * 397 * @param file File to create a Reader from. 398 * @throws NullPointerException if the input is {@code null}. 399 * @throws IOException thrown if there is a problem reading the file. 400 * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()} 401 */ 402 @Deprecated 403 public XmlStreamReader(final File file) throws IOException { 404 this(Objects.requireNonNull(file, "file").toPath()); 405 } 406 407 /** 408 * Constructs a Reader for a raw InputStream. 409 * <p> 410 * It follows the same logic used for files. 411 * </p> 412 * <p> 413 * It does a lenient charset encoding detection, check the constructor with the lenient parameter for details. 414 * </p> 415 * 416 * @param inputStream InputStream to create a Reader from. 417 * @throws NullPointerException if the input stream is {@code null}. 418 * @throws IOException thrown if there is a problem reading the stream. 419 * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()} 420 */ 421 @Deprecated 422 public XmlStreamReader(final InputStream inputStream) throws IOException { 423 this(inputStream, true); 424 } 425 426 /** 427 * Constructs a Reader for a raw InputStream. 428 * <p> 429 * It follows the same logic used for files. 430 * </p> 431 * <p> 432 * If lenient detection is indicated and the detection above fails as per specifications it then attempts the following: 433 * </p> 434 * <p> 435 * If the content type was 'text/html' it replaces it with 'text/xml' and tries the detection again. 436 * </p> 437 * <p> 438 * Else if the XML prolog had a charset encoding that encoding is used. 439 * </p> 440 * <p> 441 * Else if the content type had a charset encoding that encoding is used. 442 * </p> 443 * <p> 444 * Else 'UTF-8' is used. 445 * </p> 446 * <p> 447 * If lenient detection is indicated an XmlStreamReaderException is never thrown. 448 * </p> 449 * 450 * @param inputStream InputStream to create a Reader from. 451 * @param lenient indicates if the charset encoding detection should be relaxed. 452 * @throws NullPointerException if the input stream is {@code null}. 453 * @throws IOException thrown if there is a problem reading the stream. 454 * @throws XmlStreamReaderException thrown if the charset encoding could not be determined according to the specification. 455 * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()} 456 */ 457 @Deprecated 458 public XmlStreamReader(final InputStream inputStream, final boolean lenient) throws IOException { 459 this(inputStream, lenient, null); 460 } 461 462 /** 463 * Constructs a Reader for a raw InputStream. 464 * <p> 465 * It follows the same logic used for files. 466 * </p> 467 * <p> 468 * If lenient detection is indicated and the detection above fails as per specifications it then attempts the following: 469 * </p> 470 * <p> 471 * If the content type was 'text/html' it replaces it with 'text/xml' and tries the detection again. 472 * </p> 473 * <p> 474 * Else if the XML prolog had a charset encoding that encoding is used. 475 * </p> 476 * <p> 477 * Else if the content type had a charset encoding that encoding is used. 478 * </p> 479 * <p> 480 * Else 'UTF-8' is used. 481 * </p> 482 * <p> 483 * If lenient detection is indicated an XmlStreamReaderException is never thrown. 484 * </p> 485 * 486 * @param inputStream InputStream to create a Reader from. 487 * @param lenient indicates if the charset encoding detection should be relaxed. 488 * @param defaultEncoding The default encoding 489 * @throws NullPointerException if the input stream is {@code null}. 490 * @throws IOException thrown if there is a problem reading the stream. 491 * @throws XmlStreamReaderException thrown if the charset encoding could not be determined according to the specification. 492 * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()} 493 */ 494 @Deprecated 495 @SuppressWarnings("resource") // InputStream is managed through a InputStreamReader in this instance. 496 public XmlStreamReader(final InputStream inputStream, final boolean lenient, final String defaultEncoding) throws IOException { 497 this.defaultEncoding = defaultEncoding; 498 final BOMInputStream bom = new BOMInputStream(new BufferedInputStream(Objects.requireNonNull(inputStream, "inputStream"), IOUtils.DEFAULT_BUFFER_SIZE), 499 false, BOMS); 500 final BOMInputStream pis = new BOMInputStream(bom, true, XML_GUESS_BYTES); 501 this.encoding = processHttpStream(bom, pis, lenient); 502 this.reader = new InputStreamReader(pis, encoding); 503 } 504 505 /** 506 * Constructs a Reader using an InputStream and the associated content-type header. 507 * <p> 508 * First it checks if the stream has BOM. If there is not BOM checks the content-type encoding. If there is not content-type encoding checks the XML prolog 509 * encoding. If there is not XML prolog encoding uses the default encoding mandated by the content-type MIME type. 510 * </p> 511 * <p> 512 * It does a lenient charset encoding detection, check the constructor with the lenient parameter for details. 513 * </p> 514 * 515 * @param inputStream InputStream to create the reader from. 516 * @param httpContentType content-type header to use for the resolution of the charset encoding. 517 * @throws NullPointerException if the input stream is {@code null}. 518 * @throws IOException thrown if there is a problem reading the file. 519 * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()} 520 */ 521 @Deprecated 522 public XmlStreamReader(final InputStream inputStream, final String httpContentType) throws IOException { 523 this(inputStream, httpContentType, true); 524 } 525 526 /** 527 * Constructs a Reader using an InputStream and the associated content-type header. This constructor is lenient regarding the encoding detection. 528 * <p> 529 * First it checks if the stream has BOM. If there is not BOM checks the content-type encoding. If there is not content-type encoding checks the XML prolog 530 * encoding. If there is not XML prolog encoding uses the default encoding mandated by the content-type MIME type. 531 * </p> 532 * <p> 533 * If lenient detection is indicated and the detection above fails as per specifications it then attempts the following: 534 * </p> 535 * <p> 536 * If the content type was 'text/html' it replaces it with 'text/xml' and tries the detection again. 537 * </p> 538 * <p> 539 * Else if the XML prolog had a charset encoding that encoding is used. 540 * </p> 541 * <p> 542 * Else if the content type had a charset encoding that encoding is used. 543 * </p> 544 * <p> 545 * Else 'UTF-8' is used. 546 * </p> 547 * <p> 548 * If lenient detection is indicated an XmlStreamReaderException is never thrown. 549 * </p> 550 * 551 * @param inputStream InputStream to create the reader from. 552 * @param httpContentType content-type header to use for the resolution of the charset encoding. 553 * @param lenient indicates if the charset encoding detection should be relaxed. 554 * @throws NullPointerException if the input stream is {@code null}. 555 * @throws IOException thrown if there is a problem reading the file. 556 * @throws XmlStreamReaderException thrown if the charset encoding could not be determined according to the specification. 557 * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()} 558 */ 559 @Deprecated 560 public XmlStreamReader(final InputStream inputStream, final String httpContentType, final boolean lenient) throws IOException { 561 this(inputStream, httpContentType, lenient, null); 562 } 563 564 /** 565 * Constructs a Reader using an InputStream and the associated content-type header. This constructor is lenient regarding the encoding detection. 566 * <p> 567 * First it checks if the stream has BOM. If there is not BOM checks the content-type encoding. If there is not content-type encoding checks the XML prolog 568 * encoding. If there is not XML prolog encoding uses the default encoding mandated by the content-type MIME type. 569 * </p> 570 * <p> 571 * If lenient detection is indicated and the detection above fails as per specifications it then attempts the following: 572 * </p> 573 * <p> 574 * If the content type was 'text/html' it replaces it with 'text/xml' and tries the detection again. 575 * </p> 576 * <p> 577 * Else if the XML prolog had a charset encoding that encoding is used. 578 * </p> 579 * <p> 580 * Else if the content type had a charset encoding that encoding is used. 581 * </p> 582 * <p> 583 * Else 'UTF-8' is used. 584 * </p> 585 * <p> 586 * If lenient detection is indicated an XmlStreamReaderException is never thrown. 587 * </p> 588 * 589 * @param inputStream InputStream to create the reader from. 590 * @param httpContentType content-type header to use for the resolution of the charset encoding. 591 * @param lenient indicates if the charset encoding detection should be relaxed. 592 * @param defaultEncoding The default encoding 593 * @throws NullPointerException if the input stream is {@code null}. 594 * @throws IOException thrown if there is a problem reading the file. 595 * @throws XmlStreamReaderException thrown if the charset encoding could not be determined according to the specification. 596 * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()} 597 */ 598 @Deprecated 599 @SuppressWarnings("resource") // InputStream is managed through a InputStreamReader in this instance. 600 public XmlStreamReader(final InputStream inputStream, final String httpContentType, final boolean lenient, final String defaultEncoding) 601 throws IOException { 602 this.defaultEncoding = defaultEncoding; 603 final BOMInputStream bom = new BOMInputStream(new BufferedInputStream(Objects.requireNonNull(inputStream, "inputStream"), IOUtils.DEFAULT_BUFFER_SIZE), 604 false, BOMS); 605 final BOMInputStream pis = new BOMInputStream(bom, true, XML_GUESS_BYTES); 606 this.encoding = processHttpStream(bom, pis, lenient, httpContentType); 607 this.reader = new InputStreamReader(pis, encoding); 608 } 609 610 /** 611 * Constructs a Reader for a File. 612 * <p> 613 * It looks for the UTF-8 BOM first, if none sniffs the XML prolog charset, if this is also missing defaults to UTF-8. 614 * </p> 615 * <p> 616 * It does a lenient charset encoding detection, check the constructor with the lenient parameter for details. 617 * </p> 618 * 619 * @param file File to create a Reader from. 620 * @throws NullPointerException if the input is {@code null}. 621 * @throws IOException thrown if there is a problem reading the file. 622 * @since 2.11.0 623 * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()} 624 */ 625 @Deprecated 626 @SuppressWarnings("resource") // InputStream is managed through another reader in this instance. 627 public XmlStreamReader(final Path file) throws IOException { 628 this(Files.newInputStream(Objects.requireNonNull(file, "file"))); 629 } 630 631 /** 632 * Constructs a Reader using the InputStream of a URL. 633 * <p> 634 * If the URL is not of type HTTP and there is not 'content-type' header in the fetched data it uses the same logic used for Files. 635 * </p> 636 * <p> 637 * If the URL is a HTTP Url or there is a 'content-type' header in the fetched data it uses the same logic used for an InputStream with content-type. 638 * </p> 639 * <p> 640 * It does a lenient charset encoding detection, check the constructor with the lenient parameter for details. 641 * </p> 642 * 643 * @param url URL to create a Reader from. 644 * @throws NullPointerException if the input is {@code null}. 645 * @throws IOException thrown if there is a problem reading the stream of the URL. 646 */ 647 public XmlStreamReader(final URL url) throws IOException { 648 this(Objects.requireNonNull(url, "url").openConnection(), null); 649 } 650 651 /** 652 * Constructs a Reader using the InputStream of a URLConnection. 653 * <p> 654 * If the URLConnection is not of type HttpURLConnection and there is not 'content-type' header in the fetched data it uses the same logic used for files. 655 * </p> 656 * <p> 657 * If the URLConnection is a HTTP Url or there is a 'content-type' header in the fetched data it uses the same logic used for an InputStream with 658 * content-type. 659 * </p> 660 * <p> 661 * It does a lenient charset encoding detection, check the constructor with the lenient parameter for details. 662 * </p> 663 * 664 * @param urlConnection URLConnection to create a Reader from. 665 * @param defaultEncoding The default encoding 666 * @throws NullPointerException if the input is {@code null}. 667 * @throws IOException thrown if there is a problem reading the stream of the URLConnection. 668 */ 669 public XmlStreamReader(final URLConnection urlConnection, final String defaultEncoding) throws IOException { 670 Objects.requireNonNull(urlConnection, "urlConnection"); 671 this.defaultEncoding = defaultEncoding; 672 final boolean lenient = true; 673 final String contentType = urlConnection.getContentType(); 674 final InputStream inputStream = urlConnection.getInputStream(); 675 @SuppressWarnings("resource") // managed by the InputStreamReader tracked by this instance 676 // @formatter:off 677 final BOMInputStream bomInput = BOMInputStream.builder() 678 .setInputStream(new BufferedInputStream(inputStream, IOUtils.DEFAULT_BUFFER_SIZE)) 679 .setInclude(false) 680 .setByteOrderMarks(BOMS) 681 .get(); 682 @SuppressWarnings("resource") 683 final BOMInputStream piInput = BOMInputStream.builder() 684 .setInputStream(new BufferedInputStream(bomInput, IOUtils.DEFAULT_BUFFER_SIZE)) 685 .setInclude(true) 686 .setByteOrderMarks(XML_GUESS_BYTES) 687 .get(); 688 // @formatter:on 689 if (urlConnection instanceof HttpURLConnection || contentType != null) { 690 this.encoding = processHttpStream(bomInput, piInput, lenient, contentType); 691 } else { 692 this.encoding = processHttpStream(bomInput, piInput, lenient); 693 } 694 this.reader = new InputStreamReader(piInput, encoding); 695 } 696 697 /** 698 * Calculates the HTTP encoding. 699 * @param bomEnc BOM encoding 700 * @param xmlGuessEnc XML Guess encoding 701 * @param xmlEnc XML encoding 702 * @param lenient indicates if the charset encoding detection should be relaxed. 703 * @param httpContentType The HTTP content type 704 * @return the HTTP encoding 705 * @throws IOException thrown if there is a problem reading the stream. 706 */ 707 String calculateHttpEncoding(final String bomEnc, final String xmlGuessEnc, final String xmlEnc, final boolean lenient, final String httpContentType) 708 throws IOException { 709 710 // Lenient and has XML encoding 711 if (lenient && xmlEnc != null) { 712 return xmlEnc; 713 } 714 715 // Determine mime/encoding content types from HTTP Content Type 716 final String cTMime = getContentTypeMime(httpContentType); 717 final String cTEnc = getContentTypeEncoding(httpContentType); 718 final boolean appXml = isAppXml(cTMime); 719 final boolean textXml = isTextXml(cTMime); 720 721 // Mime type NOT "application/xml" or "text/xml" 722 if (!appXml && !textXml) { 723 final String msg = MessageFormat.format(HTTP_EX_3, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc); 724 throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc); 725 } 726 727 // No content type encoding 728 if (cTEnc == null) { 729 if (appXml) { 730 return calculateRawEncoding(bomEnc, xmlGuessEnc, xmlEnc); 731 } 732 return defaultEncoding == null ? US_ASCII : defaultEncoding; 733 } 734 735 // UTF-16BE or UTF-16LE content type encoding 736 if (cTEnc.equals(UTF_16BE) || cTEnc.equals(UTF_16LE)) { 737 if (bomEnc != null) { 738 final String msg = MessageFormat.format(HTTP_EX_1, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc); 739 throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc); 740 } 741 return cTEnc; 742 } 743 744 // UTF-16 content type encoding 745 if (cTEnc.equals(UTF_16)) { 746 if (bomEnc != null && bomEnc.startsWith(UTF_16)) { 747 return bomEnc; 748 } 749 final String msg = MessageFormat.format(HTTP_EX_2, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc); 750 throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc); 751 } 752 753 // UTF-32BE or UTF-132E content type encoding 754 if (cTEnc.equals(UTF_32BE) || cTEnc.equals(UTF_32LE)) { 755 if (bomEnc != null) { 756 final String msg = MessageFormat.format(HTTP_EX_1, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc); 757 throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc); 758 } 759 return cTEnc; 760 } 761 762 // UTF-32 content type encoding 763 if (cTEnc.equals(UTF_32)) { 764 if (bomEnc != null && bomEnc.startsWith(UTF_32)) { 765 return bomEnc; 766 } 767 final String msg = MessageFormat.format(HTTP_EX_2, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc); 768 throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc); 769 } 770 771 return cTEnc; 772 } 773 774 /** 775 * Calculate the raw encoding. 776 * 777 * @param bomEnc BOM encoding 778 * @param xmlGuessEnc XML Guess encoding 779 * @param xmlEnc XML encoding 780 * @return the raw encoding 781 * @throws IOException thrown if there is a problem reading the stream. 782 */ 783 String calculateRawEncoding(final String bomEnc, final String xmlGuessEnc, final String xmlEnc) throws IOException { 784 785 // BOM is Null 786 if (bomEnc == null) { 787 if (xmlGuessEnc == null || xmlEnc == null) { 788 return defaultEncoding == null ? UTF_8 : defaultEncoding; 789 } 790 if (xmlEnc.equals(UTF_16) && (xmlGuessEnc.equals(UTF_16BE) || xmlGuessEnc.equals(UTF_16LE))) { 791 return xmlGuessEnc; 792 } 793 return xmlEnc; 794 } 795 796 // BOM is UTF-8 797 if (bomEnc.equals(UTF_8)) { 798 if (xmlGuessEnc != null && !xmlGuessEnc.equals(UTF_8)) { 799 final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc); 800 throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc); 801 } 802 if (xmlEnc != null && !xmlEnc.equals(UTF_8)) { 803 final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc); 804 throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc); 805 } 806 return bomEnc; 807 } 808 809 // BOM is UTF-16BE or UTF-16LE 810 if (bomEnc.equals(UTF_16BE) || bomEnc.equals(UTF_16LE)) { 811 if (xmlGuessEnc != null && !xmlGuessEnc.equals(bomEnc)) { 812 final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc); 813 throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc); 814 } 815 if (xmlEnc != null && !xmlEnc.equals(UTF_16) && !xmlEnc.equals(bomEnc)) { 816 final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc); 817 throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc); 818 } 819 return bomEnc; 820 } 821 822 // BOM is UTF-32BE or UTF-32LE 823 if (bomEnc.equals(UTF_32BE) || bomEnc.equals(UTF_32LE)) { 824 if (xmlGuessEnc != null && !xmlGuessEnc.equals(bomEnc)) { 825 final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc); 826 throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc); 827 } 828 if (xmlEnc != null && !xmlEnc.equals(UTF_32) && !xmlEnc.equals(bomEnc)) { 829 final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc); 830 throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc); 831 } 832 return bomEnc; 833 } 834 835 // BOM is something else 836 final String msg = MessageFormat.format(RAW_EX_2, bomEnc, xmlGuessEnc, xmlEnc); 837 throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc); 838 } 839 840 /** 841 * Closes the XmlStreamReader stream. 842 * 843 * @throws IOException thrown if there was a problem closing the stream. 844 */ 845 @Override 846 public void close() throws IOException { 847 reader.close(); 848 } 849 850 /** 851 * Does lenient detection. 852 * 853 * @param httpContentType content-type header to use for the resolution of the charset encoding. 854 * @param ex The thrown exception 855 * @return the encoding 856 * @throws IOException thrown if there is a problem reading the stream. 857 */ 858 private String doLenientDetection(String httpContentType, XmlStreamReaderException ex) throws IOException { 859 if (httpContentType != null && httpContentType.startsWith("text/html")) { 860 httpContentType = httpContentType.substring("text/html".length()); 861 httpContentType = "text/xml" + httpContentType; 862 try { 863 return calculateHttpEncoding(ex.getBomEncoding(), ex.getXmlGuessEncoding(), ex.getXmlEncoding(), true, httpContentType); 864 } catch (final XmlStreamReaderException ex2) { 865 ex = ex2; 866 } 867 } 868 String encoding = ex.getXmlEncoding(); 869 if (encoding == null) { 870 encoding = ex.getContentTypeEncoding(); 871 } 872 if (encoding == null) { 873 encoding = defaultEncoding == null ? UTF_8 : defaultEncoding; 874 } 875 return encoding; 876 } 877 878 /** 879 * Gets the default encoding to use if none is set in HTTP content-type, XML prolog and the rules based on content-type are not adequate. 880 * <p> 881 * If it is {@code null} the content-type based rules are used. 882 * </p> 883 * 884 * @return the default encoding to use. 885 */ 886 public String getDefaultEncoding() { 887 return defaultEncoding; 888 } 889 890 /** 891 * Gets the charset encoding of the XmlStreamReader. 892 * 893 * @return charset encoding. 894 */ 895 public String getEncoding() { 896 return encoding; 897 } 898 899 /** 900 * Process the raw stream. 901 * 902 * @param bomInput BOMInputStream to detect byte order marks 903 * @param piInput BOMInputStream to guess XML encoding 904 * @param lenient indicates if the charset encoding detection should be relaxed. 905 * @return the encoding to be used 906 * @throws IOException thrown if there is a problem reading the stream. 907 */ 908 private String processHttpStream(final BOMInputStream bomInput, final BOMInputStream piInput, final boolean lenient) throws IOException { 909 final String bomEnc = bomInput.getBOMCharsetName(); 910 final String xmlGuessEnc = piInput.getBOMCharsetName(); 911 final String xmlEnc = getXmlProlog(piInput, xmlGuessEnc); 912 try { 913 return calculateRawEncoding(bomEnc, xmlGuessEnc, xmlEnc); 914 } catch (final XmlStreamReaderException ex) { 915 if (lenient) { 916 return doLenientDetection(null, ex); 917 } 918 throw ex; 919 } 920 } 921 922 /** 923 * Processes an HTTP stream. 924 * 925 * @param bomInput BOMInputStream to detect byte order marks 926 * @param piInput BOMInputStream to guess XML encoding 927 * @param lenient indicates if the charset encoding detection should be relaxed. 928 * @param httpContentType The HTTP content type 929 * @return the encoding to be used 930 * @throws IOException thrown if there is a problem reading the stream. 931 */ 932 private String processHttpStream(final BOMInputStream bomInput, final BOMInputStream piInput, final boolean lenient, final String httpContentType) 933 throws IOException { 934 final String bomEnc = bomInput.getBOMCharsetName(); 935 final String xmlGuessEnc = piInput.getBOMCharsetName(); 936 final String xmlEnc = getXmlProlog(piInput, xmlGuessEnc); 937 try { 938 return calculateHttpEncoding(bomEnc, xmlGuessEnc, xmlEnc, lenient, httpContentType); 939 } catch (final XmlStreamReaderException ex) { 940 if (lenient) { 941 return doLenientDetection(httpContentType, ex); 942 } 943 throw ex; 944 } 945 } 946 947 /** 948 * Reads the underlying reader's {@code read(char[], int, int)} method. 949 * 950 * @param buf the buffer to read the characters into 951 * @param offset The start offset 952 * @param len The number of bytes to read 953 * @return the number of characters read or -1 if the end of stream 954 * @throws IOException if an I/O error occurs. 955 */ 956 @Override 957 public int read(final char[] buf, final int offset, final int len) throws IOException { 958 return reader.read(buf, offset, len); 959 } 960 961}