001/*
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 *      https://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017package org.apache.commons.io.input;
018
019import java.io.BufferedInputStream;
020import java.io.BufferedReader;
021import java.io.File;
022import java.io.IOException;
023import java.io.InputStream;
024import java.io.InputStreamReader;
025import java.io.Reader;
026import java.io.StringReader;
027import java.net.HttpURLConnection;
028import java.net.URL;
029import java.net.URLConnection;
030import java.nio.charset.Charset;
031import java.nio.charset.StandardCharsets;
032import java.nio.file.Files;
033import java.nio.file.Path;
034import java.text.MessageFormat;
035import java.util.Locale;
036import java.util.Objects;
037import java.util.regex.Matcher;
038import java.util.regex.Pattern;
039
040import org.apache.commons.io.ByteOrderMark;
041import org.apache.commons.io.Charsets;
042import org.apache.commons.io.IOUtils;
043import org.apache.commons.io.build.AbstractStreamBuilder;
044import org.apache.commons.io.function.IOConsumer;
045
046/**
047 * Character stream that handles all the necessary Voodoo to figure out the charset encoding of the XML document within the stream.
048 * <p>
049 * IMPORTANT: This class is not related in any way to the org.xml.sax.XMLReader. This one IS a character stream.
050 * </p>
051 * <p>
052 * All this has to be done without consuming characters from the stream, if not the XML parser will not recognized the document as a valid XML. This is not 100%
053 * true, but it's close enough (UTF-8 BOM is not handled by all parsers right now, XmlStreamReader handles it and things work in all parsers).
054 * </p>
055 * <p>
056 * The XmlStreamReader class handles the charset encoding of XML documents in Files, raw streams and HTTP streams by offering a wide set of constructors.
057 * </p>
058 * <p>
059 * By default the charset encoding detection is lenient, the constructor with the lenient flag can be used for a script (following HTTP MIME and XML
060 * specifications). All this is nicely explained by Mark Pilgrim in his blog, <a href="https://diveintomark.org/archives/2004/02/13/xml-media-types">
061 * Determining the character encoding of a feed</a>.
062 * </p>
063 * <p>
064 * To build an instance, use {@link Builder}.
065 * </p>
066 * <p>
067 * Originally developed for <a href="https://rome.dev.java.net">ROME</a> under Apache License 2.0.
068 * </p>
069 *
070 * @see Builder
071 * @see org.apache.commons.io.output.XmlStreamWriter
072 * @since 2.0
073 */
074public class XmlStreamReader extends Reader {
075
076    // @formatter:off
077    /**
078     * Builds a new {@link XmlStreamReader}.
079     *
080     * Constructs a Reader using an InputStream and the associated content-type header. This constructor is lenient regarding the encoding detection.
081     * <p>
082     * First it checks if the stream has BOM. If there is not BOM checks the content-type encoding. If there is not content-type encoding checks the XML prolog
083     * encoding. If there is not XML prolog encoding uses the default encoding mandated by the content-type MIME type.
084     * </p>
085     * <p>
086     * If lenient detection is indicated and the detection above fails as per specifications it then attempts the following:
087     * </p>
088     * <p>
089     * If the content type was 'text/html' it replaces it with 'text/xml' and tries the detection again.
090     * </p>
091     * <p>
092     * Else if the XML prolog had a charset encoding that encoding is used.
093     * </p>
094     * <p>
095     * Else if the content type had a charset encoding that encoding is used.
096     * </p>
097     * <p>
098     * Else 'UTF-8' is used.
099     * </p>
100     * <p>
101     * If lenient detection is indicated an XmlStreamReaderException is never thrown.
102     * </p>
103     * <p>
104     * For example:
105     * </p>
106     *
107     * <pre>{@code
108     * XmlStreamReader r = XmlStreamReader.builder()
109     *   .setPath(path)
110     *   .setCharset(StandardCharsets.UTF_8)
111     *   .get();
112     * }
113     * </pre>
114     *
115     * @see #get()
116     * @since 2.12.0
117     */
118    // @formatter:on
119    public static class Builder extends AbstractStreamBuilder<XmlStreamReader, Builder> {
120
121        private boolean nullCharset = true;
122        private boolean lenient = true;
123        private String httpContentType;
124
125        /**
126         * Constructs a new builder of {@link XmlStreamReader}.
127         */
128        public Builder() {
129            // empty
130        }
131
132        /**
133         * Builds a new {@link XmlStreamReader}.
134         * <p>
135         * You must set an aspect that supports {@link #getInputStream()}, otherwise, this method throws an exception.
136         * </p>
137         * <p>
138         * This builder uses the following aspects:
139         * </p>
140         * <ul>
141         * <li>{@link #getInputStream()}</li>
142         * <li>{@link #getCharset()}</li>
143         * <li>lenient</li>
144         * <li>httpContentType</li>
145         * </ul>
146         *
147         * @return a new instance.
148         * @throws IllegalStateException         if the {@code origin} is {@code null}.
149         * @throws UnsupportedOperationException if the origin cannot be converted to an {@link InputStream}.
150         * @throws IOException                   if an I/O error occurs converting to an {@link InputStream} using {@link #getInputStream()}.
151         * @throws XmlStreamReaderException thrown if the Charset encoding could not be determined according to the specification.
152         * @see #getInputStream()
153         * @see #getUnchecked()
154         */
155        @Override
156        public XmlStreamReader get() throws IOException {
157            final String defaultEncoding = nullCharset ? null : getCharset().name();
158            // @formatter:off
159            return httpContentType == null
160                    ? new XmlStreamReader(getInputStream(), lenient, defaultEncoding)
161                    : new XmlStreamReader(getInputStream(), httpContentType, lenient, defaultEncoding);
162            // @formatter:on
163        }
164
165        @Override
166        public Builder setCharset(final Charset charset) {
167            nullCharset = charset == null;
168            return super.setCharset(charset);
169        }
170
171        @Override
172        public Builder setCharset(final String charset) {
173            nullCharset = charset == null;
174            return super.setCharset(Charsets.toCharset(charset, getCharsetDefault()));
175        }
176
177        /**
178         * Sets the HTTP content type.
179         *
180         * @param httpContentType the HTTP content type.
181         * @return {@code this} instance.
182         */
183        public Builder setHttpContentType(final String httpContentType) {
184            this.httpContentType = httpContentType;
185            return this;
186        }
187
188        /**
189         * Sets the lenient toggle.
190         *
191         * @param lenient the lenient toggle.
192         * @return {@code this} instance.
193         */
194        public Builder setLenient(final boolean lenient) {
195            this.lenient = lenient;
196            return this;
197        }
198
199    }
200
201    private static final String UTF_8 = StandardCharsets.UTF_8.name();
202
203    private static final String US_ASCII = StandardCharsets.US_ASCII.name();
204
205    private static final String UTF_16BE = StandardCharsets.UTF_16BE.name();
206
207    private static final String UTF_16LE = StandardCharsets.UTF_16LE.name();
208
209    private static final String UTF_32BE = "UTF-32BE";
210
211    private static final String UTF_32LE = "UTF-32LE";
212
213    private static final String UTF_16 = StandardCharsets.UTF_16.name();
214
215    private static final String UTF_32 = "UTF-32";
216
217    private static final String EBCDIC = "CP1047";
218
219    private static final ByteOrderMark[] BOMS = { ByteOrderMark.UTF_8, ByteOrderMark.UTF_16BE, ByteOrderMark.UTF_16LE, ByteOrderMark.UTF_32BE,
220            ByteOrderMark.UTF_32LE };
221
222    /** UTF_16LE and UTF_32LE have the same two starting BOM bytes. */
223    private static final ByteOrderMark[] XML_GUESS_BYTES = { new ByteOrderMark(UTF_8, 0x3C, 0x3F, 0x78, 0x6D),
224            new ByteOrderMark(UTF_16BE, 0x00, 0x3C, 0x00, 0x3F), new ByteOrderMark(UTF_16LE, 0x3C, 0x00, 0x3F, 0x00),
225            new ByteOrderMark(UTF_32BE, 0x00, 0x00, 0x00, 0x3C, 0x00, 0x00, 0x00, 0x3F, 0x00, 0x00, 0x00, 0x78, 0x00, 0x00, 0x00, 0x6D),
226            new ByteOrderMark(UTF_32LE, 0x3C, 0x00, 0x00, 0x00, 0x3F, 0x00, 0x00, 0x00, 0x78, 0x00, 0x00, 0x00, 0x6D, 0x00, 0x00, 0x00),
227            new ByteOrderMark(EBCDIC, 0x4C, 0x6F, 0xA7, 0x94) };
228
229    private static final Pattern CHARSET_PATTERN = Pattern.compile("charset=[\"']?([.[^; \"']]*)[\"']?");
230
231    /**
232     * Pattern capturing the encoding of the <a href="https://www.w3.org/TR/REC-xml/#sec-pi">{@code 'xml'} processing instruction</a>.
233     * <p>
234     * See also the <a href="https://www.w3.org/TR/2008/REC-xml-20081126/#NT-EncName">NT-EncName</a> XML specification.
235     * </p>
236     * <p>
237     * Note the documented pattern is:
238     * </p>
239     * <pre>
240     * EncName   ::=   [A-Za-z] ([A-Za-z0-9._] | '-')*
241     * </pre>
242     * <p>
243     * However this does not match all the aliases that are supported by Java. For example, {@code '437'}, {@code 'ISO_8859-1:1987'} and
244     * {@code 'ebcdic-de-273+euro'}.
245     * </p>
246     */
247    public static final Pattern ENCODING_PATTERN = Pattern.compile(
248    // @formatter:off
249            "^<\\?xml\\s+"
250            + "(?:version\\s*=\\s*(?:(?:\"1\\.[0-9]+\")|(?:'1.[0-9]+'))\\s+)??"
251            + "encoding\\s*=\\s*"
252            + "((?:\"[A-Za-z0-9][A-Za-z0-9._+:-]*\")"  // double-quoted
253            +  "|(?:'[A-Za-z0-9][A-Za-z0-9._+:-]*'))", // single-quoted
254            Pattern.MULTILINE);
255    // @formatter:on
256
257    private static final String RAW_EX_1 = "Illegal encoding, BOM [{0}] XML guess [{1}] XML prolog [{2}] encoding mismatch";
258
259    private static final String RAW_EX_2 = "Illegal encoding, BOM [{0}] XML guess [{1}] XML prolog [{2}] unknown BOM";
260
261    private static final String HTTP_EX_1 = "Illegal encoding, CT-MIME [{0}] CT-Enc [{1}] BOM [{2}] XML guess [{3}] XML prolog [{4}], BOM must be null";
262
263    private static final String HTTP_EX_2 = "Illegal encoding, CT-MIME [{0}] CT-Enc [{1}] BOM [{2}] XML guess [{3}] XML prolog [{4}], encoding mismatch";
264
265    private static final String HTTP_EX_3 = "Illegal encoding, CT-MIME [{0}] CT-Enc [{1}] BOM [{2}] XML guess [{3}] XML prolog [{4}], Illegal MIME";
266
267    /**
268     * Constructs a new {@link Builder}.
269     *
270     * @return a new {@link Builder}.
271     * @since 2.12.0
272     */
273    public static Builder builder() {
274        return new Builder();
275    }
276
277    /**
278     * Gets the charset parameter value, {@code null} if not present, {@code null} if httpContentType is {@code null}.
279     *
280     * @param httpContentType the HTTP content type
281     * @return The content type encoding (upcased)
282     */
283    static String getContentTypeEncoding(final String httpContentType) {
284        String encoding = null;
285        if (httpContentType != null) {
286            final int i = httpContentType.indexOf(";");
287            if (i > -1) {
288                final String postMime = httpContentType.substring(i + 1);
289                final Matcher m = CHARSET_PATTERN.matcher(postMime);
290                encoding = m.find() ? m.group(1) : null;
291                encoding = encoding != null ? encoding.toUpperCase(Locale.ROOT) : null;
292            }
293        }
294        return encoding;
295    }
296
297    /**
298     * Gets the MIME type or {@code null} if httpContentType is {@code null}.
299     *
300     * @param httpContentType the HTTP content type
301     * @return The mime content type
302     */
303    static String getContentTypeMime(final String httpContentType) {
304        String mime = null;
305        if (httpContentType != null) {
306            final int i = httpContentType.indexOf(";");
307            mime = i >= 0 ? httpContentType.substring(0, i) : httpContentType;
308            mime = mime.trim();
309        }
310        return mime;
311    }
312
313    /**
314     * Gets the encoding declared in the <?xml encoding=...?>, {@code null} if none.
315     *
316     * @param inputStream InputStream to create the reader from.
317     * @param guessedEnc  guessed encoding
318     * @return the encoding declared in the <?xml encoding=...?>
319     * @throws IOException thrown if there is a problem reading the stream.
320     */
321    private static String getXmlProlog(final InputStream inputStream, final String guessedEnc) throws IOException {
322        String encoding = null;
323        if (guessedEnc != null) {
324            final byte[] bytes = IOUtils.byteArray();
325            inputStream.mark(IOUtils.DEFAULT_BUFFER_SIZE);
326            int offset = 0;
327            int max = IOUtils.DEFAULT_BUFFER_SIZE;
328            int c = inputStream.read(bytes, offset, max);
329            int firstGT = -1;
330            String xmlProlog = ""; // avoid possible NPE warning (cannot happen; this just silences the warning)
331            while (c != -1 && firstGT == -1 && offset < IOUtils.DEFAULT_BUFFER_SIZE) {
332                offset += c;
333                max -= c;
334                c = inputStream.read(bytes, offset, max);
335                xmlProlog = new String(bytes, 0, offset, guessedEnc);
336                firstGT = xmlProlog.indexOf('>');
337            }
338            if (firstGT == -1) {
339                if (c == -1) {
340                    throw new IOException("Unexpected end of XML stream");
341                }
342                throw new IOException("XML prolog or ROOT element not found on first " + offset + " bytes");
343            }
344            final int bytesRead = offset;
345            if (bytesRead > 0) {
346                inputStream.reset();
347                final BufferedReader bReader = new BufferedReader(new StringReader(xmlProlog.substring(0, firstGT + 1)));
348                final StringBuilder prolog = new StringBuilder();
349                IOConsumer.forEach(bReader.lines(), l -> prolog.append(l).append(' '));
350                final Matcher m = ENCODING_PATTERN.matcher(prolog);
351                if (m.find()) {
352                    encoding = m.group(1).toUpperCase(Locale.ROOT);
353                    encoding = encoding.substring(1, encoding.length() - 1);
354                }
355            }
356        }
357        return encoding;
358    }
359
360    /**
361     * Tests if the MIME type belongs to the APPLICATION XML family.
362     *
363     * @param mime The mime type
364     * @return true if the mime type belongs to the APPLICATION XML family, otherwise false
365     */
366    static boolean isAppXml(final String mime) {
367        return mime != null && (mime.equals("application/xml") || mime.equals("application/xml-dtd") || mime.equals("application/xml-external-parsed-entity")
368                || mime.startsWith("application/") && mime.endsWith("+xml"));
369    }
370
371    /**
372     * Tests if the MIME type belongs to the TEXT XML family.
373     *
374     * @param mime The mime type
375     * @return true if the mime type belongs to the TEXT XML family, otherwise false
376     */
377    static boolean isTextXml(final String mime) {
378        return mime != null && (mime.equals("text/xml") || mime.equals("text/xml-external-parsed-entity") || mime.startsWith("text/") && mime.endsWith("+xml"));
379    }
380
381    private final Reader reader;
382
383    private final String encoding;
384
385    private final String defaultEncoding;
386
387    /**
388     * Constructs a Reader for a File.
389     * <p>
390     * It looks for the UTF-8 BOM first, if none sniffs the XML prolog charset, if this is also missing defaults to UTF-8.
391     * </p>
392     * <p>
393     * It does a lenient charset encoding detection, check the constructor with the lenient parameter for details.
394     * </p>
395     *
396     * @param file File to create a Reader from.
397     * @throws NullPointerException if the input is {@code null}.
398     * @throws IOException          thrown if there is a problem reading the file.
399     * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()}
400     */
401    @Deprecated
402    public XmlStreamReader(final File file) throws IOException {
403        this(Objects.requireNonNull(file, "file").toPath());
404    }
405
406    /**
407     * Constructs a Reader for a raw InputStream.
408     * <p>
409     * It follows the same logic used for files.
410     * </p>
411     * <p>
412     * It does a lenient charset encoding detection, check the constructor with the lenient parameter for details.
413     * </p>
414     *
415     * @param inputStream InputStream to create a Reader from.
416     * @throws NullPointerException if the input stream is {@code null}.
417     * @throws IOException          thrown if there is a problem reading the stream.
418     * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()}
419     */
420    @Deprecated
421    public XmlStreamReader(final InputStream inputStream) throws IOException {
422        this(inputStream, true);
423    }
424
425    /**
426     * Constructs a Reader for a raw InputStream.
427     * <p>
428     * It follows the same logic used for files.
429     * </p>
430     * <p>
431     * If lenient detection is indicated and the detection above fails as per specifications it then attempts the following:
432     * </p>
433     * <p>
434     * If the content type was 'text/html' it replaces it with 'text/xml' and tries the detection again.
435     * </p>
436     * <p>
437     * Else if the XML prolog had a charset encoding that encoding is used.
438     * </p>
439     * <p>
440     * Else if the content type had a charset encoding that encoding is used.
441     * </p>
442     * <p>
443     * Else 'UTF-8' is used.
444     * </p>
445     * <p>
446     * If lenient detection is indicated an XmlStreamReaderException is never thrown.
447     * </p>
448     *
449     * @param inputStream InputStream to create a Reader from.
450     * @param lenient     indicates if the charset encoding detection should be relaxed.
451     * @throws NullPointerException     if the input stream is {@code null}.
452     * @throws IOException              thrown if there is a problem reading the stream.
453     * @throws XmlStreamReaderException thrown if the charset encoding could not be determined according to the specification.
454     * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()}
455     */
456    @Deprecated
457    public XmlStreamReader(final InputStream inputStream, final boolean lenient) throws IOException {
458        this(inputStream, lenient, null);
459    }
460
461    /**
462     * Constructs a Reader for a raw InputStream.
463     * <p>
464     * It follows the same logic used for files.
465     * </p>
466     * <p>
467     * If lenient detection is indicated and the detection above fails as per specifications it then attempts the following:
468     * </p>
469     * <p>
470     * If the content type was 'text/html' it replaces it with 'text/xml' and tries the detection again.
471     * </p>
472     * <p>
473     * Else if the XML prolog had a charset encoding that encoding is used.
474     * </p>
475     * <p>
476     * Else if the content type had a charset encoding that encoding is used.
477     * </p>
478     * <p>
479     * Else 'UTF-8' is used.
480     * </p>
481     * <p>
482     * If lenient detection is indicated an XmlStreamReaderException is never thrown.
483     * </p>
484     *
485     * @param inputStream     InputStream to create a Reader from.
486     * @param lenient         indicates if the charset encoding detection should be relaxed.
487     * @param defaultEncoding The default encoding
488     * @throws NullPointerException     if the input stream is {@code null}.
489     * @throws IOException              thrown if there is a problem reading the stream.
490     * @throws XmlStreamReaderException thrown if the charset encoding could not be determined according to the specification.
491     * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()}
492     */
493    @Deprecated
494    @SuppressWarnings("resource") // InputStream is managed through a InputStreamReader in this instance.
495    public XmlStreamReader(final InputStream inputStream, final boolean lenient, final String defaultEncoding) throws IOException {
496        this.defaultEncoding = defaultEncoding;
497        final BOMInputStream bom = new BOMInputStream(new BufferedInputStream(Objects.requireNonNull(inputStream, "inputStream"), IOUtils.DEFAULT_BUFFER_SIZE),
498                false, BOMS);
499        final BOMInputStream pis = new BOMInputStream(bom, true, XML_GUESS_BYTES);
500        this.encoding = processHttpStream(bom, pis, lenient);
501        this.reader = new InputStreamReader(pis, encoding);
502    }
503
504    /**
505     * Constructs a Reader using an InputStream and the associated content-type header.
506     * <p>
507     * First it checks if the stream has BOM. If there is not BOM checks the content-type encoding. If there is not content-type encoding checks the XML prolog
508     * encoding. If there is not XML prolog encoding uses the default encoding mandated by the content-type MIME type.
509     * </p>
510     * <p>
511     * It does a lenient charset encoding detection, check the constructor with the lenient parameter for details.
512     * </p>
513     *
514     * @param inputStream     InputStream to create the reader from.
515     * @param httpContentType content-type header to use for the resolution of the charset encoding.
516     * @throws NullPointerException if the input stream is {@code null}.
517     * @throws IOException          thrown if there is a problem reading the file.
518     * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()}
519     */
520    @Deprecated
521    public XmlStreamReader(final InputStream inputStream, final String httpContentType) throws IOException {
522        this(inputStream, httpContentType, true);
523    }
524
525    /**
526     * Constructs a Reader using an InputStream and the associated content-type header. This constructor is lenient regarding the encoding detection.
527     * <p>
528     * First it checks if the stream has BOM. If there is not BOM checks the content-type encoding. If there is not content-type encoding checks the XML prolog
529     * encoding. If there is not XML prolog encoding uses the default encoding mandated by the content-type MIME type.
530     * </p>
531     * <p>
532     * If lenient detection is indicated and the detection above fails as per specifications it then attempts the following:
533     * </p>
534     * <p>
535     * If the content type was 'text/html' it replaces it with 'text/xml' and tries the detection again.
536     * </p>
537     * <p>
538     * Else if the XML prolog had a charset encoding that encoding is used.
539     * </p>
540     * <p>
541     * Else if the content type had a charset encoding that encoding is used.
542     * </p>
543     * <p>
544     * Else 'UTF-8' is used.
545     * </p>
546     * <p>
547     * If lenient detection is indicated an XmlStreamReaderException is never thrown.
548     * </p>
549     *
550     * @param inputStream     InputStream to create the reader from.
551     * @param httpContentType content-type header to use for the resolution of the charset encoding.
552     * @param lenient         indicates if the charset encoding detection should be relaxed.
553     * @throws NullPointerException     if the input stream is {@code null}.
554     * @throws IOException              thrown if there is a problem reading the file.
555     * @throws XmlStreamReaderException thrown if the charset encoding could not be determined according to the specification.
556     * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()}
557     */
558    @Deprecated
559    public XmlStreamReader(final InputStream inputStream, final String httpContentType, final boolean lenient) throws IOException {
560        this(inputStream, httpContentType, lenient, null);
561    }
562
563    /**
564     * Constructs a Reader using an InputStream and the associated content-type header. This constructor is lenient regarding the encoding detection.
565     * <p>
566     * First it checks if the stream has BOM. If there is not BOM checks the content-type encoding. If there is not content-type encoding checks the XML prolog
567     * encoding. If there is not XML prolog encoding uses the default encoding mandated by the content-type MIME type.
568     * </p>
569     * <p>
570     * If lenient detection is indicated and the detection above fails as per specifications it then attempts the following:
571     * </p>
572     * <p>
573     * If the content type was 'text/html' it replaces it with 'text/xml' and tries the detection again.
574     * </p>
575     * <p>
576     * Else if the XML prolog had a charset encoding that encoding is used.
577     * </p>
578     * <p>
579     * Else if the content type had a charset encoding that encoding is used.
580     * </p>
581     * <p>
582     * Else 'UTF-8' is used.
583     * </p>
584     * <p>
585     * If lenient detection is indicated an XmlStreamReaderException is never thrown.
586     * </p>
587     *
588     * @param inputStream     InputStream to create the reader from.
589     * @param httpContentType content-type header to use for the resolution of the charset encoding.
590     * @param lenient         indicates if the charset encoding detection should be relaxed.
591     * @param defaultEncoding The default encoding
592     * @throws NullPointerException     if the input stream is {@code null}.
593     * @throws IOException              thrown if there is a problem reading the file.
594     * @throws XmlStreamReaderException thrown if the charset encoding could not be determined according to the specification.
595     * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()}
596     */
597    @Deprecated
598    @SuppressWarnings("resource") // InputStream is managed through a InputStreamReader in this instance.
599    public XmlStreamReader(final InputStream inputStream, final String httpContentType, final boolean lenient, final String defaultEncoding)
600            throws IOException {
601        this.defaultEncoding = defaultEncoding;
602        final BOMInputStream bom = new BOMInputStream(new BufferedInputStream(Objects.requireNonNull(inputStream, "inputStream"), IOUtils.DEFAULT_BUFFER_SIZE),
603                false, BOMS);
604        final BOMInputStream pis = new BOMInputStream(bom, true, XML_GUESS_BYTES);
605        this.encoding = processHttpStream(bom, pis, lenient, httpContentType);
606        this.reader = new InputStreamReader(pis, encoding);
607    }
608
609    /**
610     * Constructs a Reader for a File.
611     * <p>
612     * It looks for the UTF-8 BOM first, if none sniffs the XML prolog charset, if this is also missing defaults to UTF-8.
613     * </p>
614     * <p>
615     * It does a lenient charset encoding detection, check the constructor with the lenient parameter for details.
616     * </p>
617     *
618     * @param file File to create a Reader from.
619     * @throws NullPointerException if the input is {@code null}.
620     * @throws IOException          thrown if there is a problem reading the file.
621     * @since 2.11.0
622     * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()}
623     */
624    @Deprecated
625    @SuppressWarnings("resource") // InputStream is managed through another reader in this instance.
626    public XmlStreamReader(final Path file) throws IOException {
627        this(Files.newInputStream(Objects.requireNonNull(file, "file")));
628    }
629
630    /**
631     * Constructs a Reader using the InputStream of a URL.
632     * <p>
633     * If the URL is not of type HTTP and there is not 'content-type' header in the fetched data it uses the same logic used for Files.
634     * </p>
635     * <p>
636     * If the URL is a HTTP Url or there is a 'content-type' header in the fetched data it uses the same logic used for an InputStream with content-type.
637     * </p>
638     * <p>
639     * It does a lenient charset encoding detection, check the constructor with the lenient parameter for details.
640     * </p>
641     *
642     * @param url URL to create a Reader from.
643     * @throws NullPointerException if the input is {@code null}.
644     * @throws IOException          thrown if there is a problem reading the stream of the URL.
645     */
646    public XmlStreamReader(final URL url) throws IOException {
647        this(Objects.requireNonNull(url, "url").openConnection(), null);
648    }
649
650    /**
651     * Constructs a Reader using the InputStream of a URLConnection.
652     * <p>
653     * If the URLConnection is not of type HttpURLConnection and there is not 'content-type' header in the fetched data it uses the same logic used for files.
654     * </p>
655     * <p>
656     * If the URLConnection is a HTTP Url or there is a 'content-type' header in the fetched data it uses the same logic used for an InputStream with
657     * content-type.
658     * </p>
659     * <p>
660     * It does a lenient charset encoding detection, check the constructor with the lenient parameter for details.
661     * </p>
662     *
663     * @param urlConnection   URLConnection to create a Reader from.
664     * @param defaultEncoding The default encoding
665     * @throws NullPointerException if the input is {@code null}.
666     * @throws IOException          thrown if there is a problem reading the stream of the URLConnection.
667     */
668    public XmlStreamReader(final URLConnection urlConnection, final String defaultEncoding) throws IOException {
669        Objects.requireNonNull(urlConnection, "urlConnection");
670        this.defaultEncoding = defaultEncoding;
671        final boolean lenient = true;
672        final String contentType = urlConnection.getContentType();
673        final InputStream inputStream = urlConnection.getInputStream();
674        @SuppressWarnings("resource") // managed by the InputStreamReader tracked by this instance
675        // @formatter:off
676        final BOMInputStream bomInput = BOMInputStream.builder()
677            .setInputStream(new BufferedInputStream(inputStream, IOUtils.DEFAULT_BUFFER_SIZE))
678            .setInclude(false)
679            .setByteOrderMarks(BOMS)
680            .get();
681        @SuppressWarnings("resource")
682        final BOMInputStream piInput = BOMInputStream.builder()
683            .setInputStream(new BufferedInputStream(bomInput, IOUtils.DEFAULT_BUFFER_SIZE))
684            .setInclude(true)
685            .setByteOrderMarks(XML_GUESS_BYTES)
686            .get();
687        // @formatter:on
688        if (urlConnection instanceof HttpURLConnection || contentType != null) {
689            this.encoding = processHttpStream(bomInput, piInput, lenient, contentType);
690        } else {
691            this.encoding = processHttpStream(bomInput, piInput, lenient);
692        }
693        this.reader = new InputStreamReader(piInput, encoding);
694    }
695
696    /**
697     * Calculates the HTTP encoding.
698     * @param bomEnc          BOM encoding
699     * @param xmlGuessEnc     XML Guess encoding
700     * @param xmlEnc          XML encoding
701     * @param lenient         indicates if the charset encoding detection should be relaxed.
702     * @param httpContentType The HTTP content type
703     * @return the HTTP encoding
704     * @throws IOException thrown if there is a problem reading the stream.
705     */
706    String calculateHttpEncoding(final String bomEnc, final String xmlGuessEnc, final String xmlEnc, final boolean lenient, final String httpContentType)
707            throws IOException {
708
709        // Lenient and has XML encoding
710        if (lenient && xmlEnc != null) {
711            return xmlEnc;
712        }
713
714        // Determine mime/encoding content types from HTTP Content Type
715        final String cTMime = getContentTypeMime(httpContentType);
716        final String cTEnc = getContentTypeEncoding(httpContentType);
717        final boolean appXml = isAppXml(cTMime);
718        final boolean textXml = isTextXml(cTMime);
719
720        // Mime type NOT "application/xml" or "text/xml"
721        if (!appXml && !textXml) {
722            final String msg = MessageFormat.format(HTTP_EX_3, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
723            throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
724        }
725
726        // No content type encoding
727        if (cTEnc == null) {
728            if (appXml) {
729                return calculateRawEncoding(bomEnc, xmlGuessEnc, xmlEnc);
730            }
731            return defaultEncoding == null ? US_ASCII : defaultEncoding;
732        }
733
734        // UTF-16BE or UTF-16LE content type encoding
735        if (cTEnc.equals(UTF_16BE) || cTEnc.equals(UTF_16LE)) {
736            if (bomEnc != null) {
737                final String msg = MessageFormat.format(HTTP_EX_1, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
738                throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
739            }
740            return cTEnc;
741        }
742
743        // UTF-16 content type encoding
744        if (cTEnc.equals(UTF_16)) {
745            if (bomEnc != null && bomEnc.startsWith(UTF_16)) {
746                return bomEnc;
747            }
748            final String msg = MessageFormat.format(HTTP_EX_2, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
749            throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
750        }
751
752        // UTF-32BE or UTF-132E content type encoding
753        if (cTEnc.equals(UTF_32BE) || cTEnc.equals(UTF_32LE)) {
754            if (bomEnc != null) {
755                final String msg = MessageFormat.format(HTTP_EX_1, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
756                throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
757            }
758            return cTEnc;
759        }
760
761        // UTF-32 content type encoding
762        if (cTEnc.equals(UTF_32)) {
763            if (bomEnc != null && bomEnc.startsWith(UTF_32)) {
764                return bomEnc;
765            }
766            final String msg = MessageFormat.format(HTTP_EX_2, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
767            throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
768        }
769
770        return cTEnc;
771    }
772
773    /**
774     * Calculate the raw encoding.
775     *
776     * @param bomEnc      BOM encoding
777     * @param xmlGuessEnc XML Guess encoding
778     * @param xmlEnc      XML encoding
779     * @return the raw encoding
780     * @throws IOException thrown if there is a problem reading the stream.
781     */
782    String calculateRawEncoding(final String bomEnc, final String xmlGuessEnc, final String xmlEnc) throws IOException {
783
784        // BOM is Null
785        if (bomEnc == null) {
786            if (xmlGuessEnc == null || xmlEnc == null) {
787                return defaultEncoding == null ? UTF_8 : defaultEncoding;
788            }
789            if (xmlEnc.equals(UTF_16) && (xmlGuessEnc.equals(UTF_16BE) || xmlGuessEnc.equals(UTF_16LE))) {
790                return xmlGuessEnc;
791            }
792            return xmlEnc;
793        }
794
795        // BOM is UTF-8
796        if (bomEnc.equals(UTF_8)) {
797            if (xmlGuessEnc != null && !xmlGuessEnc.equals(UTF_8)) {
798                final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc);
799                throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
800            }
801            if (xmlEnc != null && !xmlEnc.equals(UTF_8)) {
802                final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc);
803                throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
804            }
805            return bomEnc;
806        }
807
808        // BOM is UTF-16BE or UTF-16LE
809        if (bomEnc.equals(UTF_16BE) || bomEnc.equals(UTF_16LE)) {
810            if (xmlGuessEnc != null && !xmlGuessEnc.equals(bomEnc)) {
811                final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc);
812                throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
813            }
814            if (xmlEnc != null && !xmlEnc.equals(UTF_16) && !xmlEnc.equals(bomEnc)) {
815                final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc);
816                throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
817            }
818            return bomEnc;
819        }
820
821        // BOM is UTF-32BE or UTF-32LE
822        if (bomEnc.equals(UTF_32BE) || bomEnc.equals(UTF_32LE)) {
823            if (xmlGuessEnc != null && !xmlGuessEnc.equals(bomEnc)) {
824                final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc);
825                throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
826            }
827            if (xmlEnc != null && !xmlEnc.equals(UTF_32) && !xmlEnc.equals(bomEnc)) {
828                final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc);
829                throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
830            }
831            return bomEnc;
832        }
833
834        // BOM is something else
835        final String msg = MessageFormat.format(RAW_EX_2, bomEnc, xmlGuessEnc, xmlEnc);
836        throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
837    }
838
839    /**
840     * Closes the XmlStreamReader stream.
841     *
842     * @throws IOException thrown if there was a problem closing the stream.
843     */
844    @Override
845    public void close() throws IOException {
846        reader.close();
847    }
848
849    /**
850     * Does lenient detection.
851     *
852     * @param httpContentType content-type header to use for the resolution of the charset encoding.
853     * @param ex              The thrown exception
854     * @return the encoding
855     * @throws IOException thrown if there is a problem reading the stream.
856     */
857    private String doLenientDetection(String httpContentType, XmlStreamReaderException ex) throws IOException {
858        if (httpContentType != null && httpContentType.startsWith("text/html")) {
859            httpContentType = httpContentType.substring("text/html".length());
860            httpContentType = "text/xml" + httpContentType;
861            try {
862                return calculateHttpEncoding(ex.getBomEncoding(), ex.getXmlGuessEncoding(), ex.getXmlEncoding(), true, httpContentType);
863            } catch (final XmlStreamReaderException ex2) {
864                ex = ex2;
865            }
866        }
867        String encoding = ex.getXmlEncoding();
868        if (encoding == null) {
869            encoding = ex.getContentTypeEncoding();
870        }
871        if (encoding == null) {
872            encoding = defaultEncoding == null ? UTF_8 : defaultEncoding;
873        }
874        return encoding;
875    }
876
877    /**
878     * Gets the default encoding to use if none is set in HTTP content-type, XML prolog and the rules based on content-type are not adequate.
879     * <p>
880     * If it is {@code null} the content-type based rules are used.
881     * </p>
882     *
883     * @return the default encoding to use.
884     */
885    public String getDefaultEncoding() {
886        return defaultEncoding;
887    }
888
889    /**
890     * Gets the charset encoding of the XmlStreamReader.
891     *
892     * @return charset encoding.
893     */
894    public String getEncoding() {
895        return encoding;
896    }
897
898    /**
899     * Process the raw stream.
900     *
901     * @param bomInput     BOMInputStream to detect byte order marks
902     * @param piInput     BOMInputStream to guess XML encoding
903     * @param lenient indicates if the charset encoding detection should be relaxed.
904     * @return the encoding to be used
905     * @throws IOException thrown if there is a problem reading the stream.
906     */
907    private String processHttpStream(final BOMInputStream bomInput, final BOMInputStream piInput, final boolean lenient) throws IOException {
908        final String bomEnc = bomInput.getBOMCharsetName();
909        final String xmlGuessEnc = piInput.getBOMCharsetName();
910        final String xmlEnc = getXmlProlog(piInput, xmlGuessEnc);
911        try {
912            return calculateRawEncoding(bomEnc, xmlGuessEnc, xmlEnc);
913        } catch (final XmlStreamReaderException ex) {
914            if (lenient) {
915                return doLenientDetection(null, ex);
916            }
917            throw ex;
918        }
919    }
920
921    /**
922     * Processes an HTTP stream.
923     *
924     * @param bomInput        BOMInputStream to detect byte order marks
925     * @param piInput         BOMInputStream to guess XML encoding
926     * @param lenient         indicates if the charset encoding detection should be relaxed.
927     * @param httpContentType The HTTP content type
928     * @return the encoding to be used
929     * @throws IOException thrown if there is a problem reading the stream.
930     */
931    private String processHttpStream(final BOMInputStream bomInput, final BOMInputStream piInput, final boolean lenient, final String httpContentType)
932            throws IOException {
933        final String bomEnc = bomInput.getBOMCharsetName();
934        final String xmlGuessEnc = piInput.getBOMCharsetName();
935        final String xmlEnc = getXmlProlog(piInput, xmlGuessEnc);
936        try {
937            return calculateHttpEncoding(bomEnc, xmlGuessEnc, xmlEnc, lenient, httpContentType);
938        } catch (final XmlStreamReaderException ex) {
939            if (lenient) {
940                return doLenientDetection(httpContentType, ex);
941            }
942            throw ex;
943        }
944    }
945
946    /**
947     * Reads the underlying reader's {@code read(char[], int, int)} method.
948     *
949     * @param buf    the buffer to read the characters into
950     * @param offset The start offset
951     * @param len    The number of bytes to read
952     * @return the number of characters read or -1 if the end of stream
953     * @throws IOException if an I/O error occurs.
954     */
955    @Override
956    public int read(final char[] buf, final int offset, final int len) throws IOException {
957        return reader.read(buf, offset, len);
958    }
959
960}