001/*
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017package org.apache.commons.io.input;
018
019import java.io.BufferedInputStream;
020import java.io.BufferedReader;
021import java.io.File;
022import java.io.FileInputStream;
023import java.io.IOException;
024import java.io.InputStream;
025import java.io.InputStreamReader;
026import java.io.Reader;
027import java.io.StringReader;
028import java.net.HttpURLConnection;
029import java.net.URL;
030import java.net.URLConnection;
031import java.text.MessageFormat;
032import java.util.Locale;
033import java.util.Objects;
034import java.util.regex.Matcher;
035import java.util.regex.Pattern;
036
037import org.apache.commons.io.ByteOrderMark;
038import org.apache.commons.io.IOUtils;
039
040/**
041 * Character stream that handles all the necessary Voodoo to figure out the
042 * charset encoding of the XML document within the stream.
043 * <p>
044 * IMPORTANT: This class is not related in any way to the org.xml.sax.XMLReader.
045 * This one IS a character stream.
046 * </p>
047 * <p>
048 * All this has to be done without consuming characters from the stream, if not
049 * the XML parser will not recognized the document as a valid XML. This is not
050 * 100% true, but it's close enough (UTF-8 BOM is not handled by all parsers
051 * right now, XmlStreamReader handles it and things work in all parsers).
052 * </p>
053 * <p>
054 * The XmlStreamReader class handles the charset encoding of XML documents in
055 * Files, raw streams and HTTP streams by offering a wide set of constructors.
056 * </p>
057 * <p>
058 * By default the charset encoding detection is lenient, the constructor with
059 * the lenient flag can be used for a script (following HTTP MIME and XML
060 * specifications). All this is nicely explained by Mark Pilgrim in his blog, <a
061 * href="http://diveintomark.org/archives/2004/02/13/xml-media-types">
062 * Determining the character encoding of a feed</a>.
063 * </p>
064 * <p>
065 * Originally developed for <a href="http://rome.dev.java.net">ROME</a> under
066 * Apache License 2.0.
067 * </p>
068 *
069 * @see org.apache.commons.io.output.XmlStreamWriter
070 * @since 2.0
071 */
072public class XmlStreamReader extends Reader {
073    private static final int BUFFER_SIZE = IOUtils.DEFAULT_BUFFER_SIZE;
074
075    private static final String UTF_8 = "UTF-8";
076
077    private static final String US_ASCII = "US-ASCII";
078
079    private static final String UTF_16BE = "UTF-16BE";
080
081    private static final String UTF_16LE = "UTF-16LE";
082
083    private static final String UTF_32BE = "UTF-32BE";
084
085    private static final String UTF_32LE = "UTF-32LE";
086
087    private static final String UTF_16 = "UTF-16";
088
089    private static final String UTF_32 = "UTF-32";
090
091    private static final String EBCDIC = "CP1047";
092
093    private static final ByteOrderMark[] BOMS = new ByteOrderMark[] {
094        ByteOrderMark.UTF_8,
095        ByteOrderMark.UTF_16BE,
096        ByteOrderMark.UTF_16LE,
097        ByteOrderMark.UTF_32BE,
098        ByteOrderMark.UTF_32LE
099    };
100
101    // UTF_16LE and UTF_32LE have the same two starting BOM bytes.
102    private static final ByteOrderMark[] XML_GUESS_BYTES = new ByteOrderMark[] {
103        new ByteOrderMark(UTF_8,    0x3C, 0x3F, 0x78, 0x6D),
104        new ByteOrderMark(UTF_16BE, 0x00, 0x3C, 0x00, 0x3F),
105        new ByteOrderMark(UTF_16LE, 0x3C, 0x00, 0x3F, 0x00),
106        new ByteOrderMark(UTF_32BE, 0x00, 0x00, 0x00, 0x3C,
107                0x00, 0x00, 0x00, 0x3F, 0x00, 0x00, 0x00, 0x78, 0x00, 0x00, 0x00, 0x6D),
108        new ByteOrderMark(UTF_32LE, 0x3C, 0x00, 0x00, 0x00,
109                0x3F, 0x00, 0x00, 0x00, 0x78, 0x00, 0x00, 0x00, 0x6D, 0x00, 0x00, 0x00),
110        new ByteOrderMark(EBCDIC,   0x4C, 0x6F, 0xA7, 0x94)
111    };
112
113    private final Reader reader;
114
115    private final String encoding;
116
117    private final String defaultEncoding;
118
119    /**
120     * Returns the default encoding to use if none is set in HTTP content-type,
121     * XML prolog and the rules based on content-type are not adequate.
122     * <p>
123     * If it is NULL the content-type based rules are used.
124     *
125     * @return the default encoding to use.
126     */
127    public String getDefaultEncoding() {
128        return defaultEncoding;
129    }
130
131    /**
132     * Creates a Reader for a File.
133     * <p>
134     * It looks for the UTF-8 BOM first, if none sniffs the XML prolog charset,
135     * if this is also missing defaults to UTF-8.
136     * <p>
137     * It does a lenient charset encoding detection, check the constructor with
138     * the lenient parameter for details.
139     *
140     * @param file File to create a Reader from.
141     * @throws IOException thrown if there is a problem reading the file.
142     */
143    public XmlStreamReader(final File file) throws IOException {
144        this(new FileInputStream(Objects.requireNonNull(file)));
145    }
146
147    /**
148     * Creates a Reader for a raw InputStream.
149     * <p>
150     * It follows the same logic used for files.
151     * <p>
152     * It does a lenient charset encoding detection, check the constructor with
153     * the lenient parameter for details.
154     *
155     * @param inputStream InputStream to create a Reader from.
156     * @throws IOException thrown if there is a problem reading the stream.
157     */
158    public XmlStreamReader(final InputStream inputStream) throws IOException {
159        this(inputStream, true);
160    }
161
162    /**
163     * Creates a Reader for a raw InputStream.
164     * <p>
165     * It follows the same logic used for files.
166     * <p>
167     * If lenient detection is indicated and the detection above fails as per
168     * specifications it then attempts the following:
169     * <p>
170     * If the content type was 'text/html' it replaces it with 'text/xml' and
171     * tries the detection again.
172     * <p>
173     * Else if the XML prolog had a charset encoding that encoding is used.
174     * <p>
175     * Else if the content type had a charset encoding that encoding is used.
176     * <p>
177     * Else 'UTF-8' is used.
178     * <p>
179     * If lenient detection is indicated an XmlStreamReaderException is never
180     * thrown.
181     *
182     * @param inputStream InputStream to create a Reader from.
183     * @param lenient indicates if the charset encoding detection should be
184     *        relaxed.
185     * @throws IOException thrown if there is a problem reading the stream.
186     * @throws XmlStreamReaderException thrown if the charset encoding could not
187     *         be determined according to the specs.
188     */
189    public XmlStreamReader(final InputStream inputStream, final boolean lenient) throws IOException {
190        this(inputStream, lenient, null);
191    }
192
193    /**
194     * Creates a Reader for a raw InputStream.
195     * <p>
196     * It follows the same logic used for files.
197     * <p>
198     * If lenient detection is indicated and the detection above fails as per
199     * specifications it then attempts the following:
200     * <p>
201     * If the content type was 'text/html' it replaces it with 'text/xml' and
202     * tries the detection again.
203     * <p>
204     * Else if the XML prolog had a charset encoding that encoding is used.
205     * <p>
206     * Else if the content type had a charset encoding that encoding is used.
207     * <p>
208     * Else 'UTF-8' is used.
209     * <p>
210     * If lenient detection is indicated an XmlStreamReaderException is never
211     * thrown.
212     *
213     * @param inputStream InputStream to create a Reader from.
214     * @param lenient indicates if the charset encoding detection should be
215     *        relaxed.
216     * @param defaultEncoding The default encoding
217     * @throws IOException thrown if there is a problem reading the stream.
218     * @throws XmlStreamReaderException thrown if the charset encoding could not
219     *         be determined according to the specs.
220     */
221    public XmlStreamReader(final InputStream inputStream, final boolean lenient, final String defaultEncoding)
222            throws IOException {
223        Objects.requireNonNull(inputStream, "inputStream");
224        this.defaultEncoding = defaultEncoding;
225        final BOMInputStream bom = new BOMInputStream(new BufferedInputStream(inputStream, BUFFER_SIZE), false, BOMS);
226        final BOMInputStream pis = new BOMInputStream(bom, true, XML_GUESS_BYTES);
227        this.encoding = doRawStream(bom, pis, lenient);
228        this.reader = new InputStreamReader(pis, encoding);
229    }
230
231    /**
232     * Creates a Reader using the InputStream of a URL.
233     * <p>
234     * If the URL is not of type HTTP and there is not 'content-type' header in
235     * the fetched data it uses the same logic used for Files.
236     * <p>
237     * If the URL is a HTTP Url or there is a 'content-type' header in the
238     * fetched data it uses the same logic used for an InputStream with
239     * content-type.
240     * <p>
241     * It does a lenient charset encoding detection, check the constructor with
242     * the lenient parameter for details.
243     *
244     * @param url URL to create a Reader from.
245     * @throws IOException thrown if there is a problem reading the stream of
246     *         the URL.
247     */
248    public XmlStreamReader(final URL url) throws IOException {
249        this(Objects.requireNonNull(url, "url").openConnection(), null);
250    }
251
252    /**
253     * Creates a Reader using the InputStream of a URLConnection.
254     * <p>
255     * If the URLConnection is not of type HttpURLConnection and there is not
256     * 'content-type' header in the fetched data it uses the same logic used for
257     * files.
258     * <p>
259     * If the URLConnection is a HTTP Url or there is a 'content-type' header in
260     * the fetched data it uses the same logic used for an InputStream with
261     * content-type.
262     * <p>
263     * It does a lenient charset encoding detection, check the constructor with
264     * the lenient parameter for details.
265     *
266     * @param conn URLConnection to create a Reader from.
267     * @param defaultEncoding The default encoding
268     * @throws IOException thrown if there is a problem reading the stream of
269     *         the URLConnection.
270     */
271    public XmlStreamReader(final URLConnection conn, final String defaultEncoding) throws IOException {
272        Objects.requireNonNull(conn, "conm");
273        this.defaultEncoding = defaultEncoding;
274        final boolean lenient = true;
275        final String contentType = conn.getContentType();
276        final InputStream inputStream = conn.getInputStream();
277        final BOMInputStream bom = new BOMInputStream(new BufferedInputStream(inputStream, BUFFER_SIZE), false, BOMS);
278        final BOMInputStream pis = new BOMInputStream(bom, true, XML_GUESS_BYTES);
279        if (conn instanceof HttpURLConnection || contentType != null) {
280            this.encoding = processHttpStream(bom, pis, contentType, lenient);
281        } else {
282            this.encoding = doRawStream(bom, pis, lenient);
283        }
284        this.reader = new InputStreamReader(pis, encoding);
285    }
286
287    /**
288     * Creates a Reader using an InputStream and the associated content-type
289     * header.
290     * <p>
291     * First it checks if the stream has BOM. If there is not BOM checks the
292     * content-type encoding. If there is not content-type encoding checks the
293     * XML prolog encoding. If there is not XML prolog encoding uses the default
294     * encoding mandated by the content-type MIME type.
295     * <p>
296     * It does a lenient charset encoding detection, check the constructor with
297     * the lenient parameter for details.
298     *
299     * @param inputStream InputStream to create the reader from.
300     * @param httpContentType content-type header to use for the resolution of
301     *        the charset encoding.
302     * @throws IOException thrown if there is a problem reading the file.
303     */
304    public XmlStreamReader(final InputStream inputStream, final String httpContentType)
305            throws IOException {
306        this(inputStream, httpContentType, true);
307    }
308
309    /**
310     * Creates a Reader using an InputStream and the associated content-type
311     * header. This constructor is lenient regarding the encoding detection.
312     * <p>
313     * First it checks if the stream has BOM. If there is not BOM checks the
314     * content-type encoding. If there is not content-type encoding checks the
315     * XML prolog encoding. If there is not XML prolog encoding uses the default
316     * encoding mandated by the content-type MIME type.
317     * <p>
318     * If lenient detection is indicated and the detection above fails as per
319     * specifications it then attempts the following:
320     * <p>
321     * If the content type was 'text/html' it replaces it with 'text/xml' and
322     * tries the detection again.
323     * <p>
324     * Else if the XML prolog had a charset encoding that encoding is used.
325     * <p>
326     * Else if the content type had a charset encoding that encoding is used.
327     * <p>
328     * Else 'UTF-8' is used.
329     * <p>
330     * If lenient detection is indicated an XmlStreamReaderException is never
331     * thrown.
332     *
333     * @param inputStream InputStream to create the reader from.
334     * @param httpContentType content-type header to use for the resolution of
335     *        the charset encoding.
336     * @param lenient indicates if the charset encoding detection should be
337     *        relaxed.
338     * @param defaultEncoding The default encoding
339     * @throws IOException thrown if there is a problem reading the file.
340     * @throws XmlStreamReaderException thrown if the charset encoding could not
341     *         be determined according to the specs.
342     */
343    public XmlStreamReader(final InputStream inputStream, final String httpContentType,
344            final boolean lenient, final String defaultEncoding) throws IOException {
345        Objects.requireNonNull(inputStream, "inputStream");
346        this.defaultEncoding = defaultEncoding;
347        final BOMInputStream bom = new BOMInputStream(new BufferedInputStream(inputStream, BUFFER_SIZE), false, BOMS);
348        final BOMInputStream pis = new BOMInputStream(bom, true, XML_GUESS_BYTES);
349        this.encoding = processHttpStream(bom, pis, httpContentType, lenient);
350        this.reader = new InputStreamReader(pis, encoding);
351    }
352
353    /**
354     * Creates a Reader using an InputStream and the associated content-type
355     * header. This constructor is lenient regarding the encoding detection.
356     * <p>
357     * First it checks if the stream has BOM. If there is not BOM checks the
358     * content-type encoding. If there is not content-type encoding checks the
359     * XML prolog encoding. If there is not XML prolog encoding uses the default
360     * encoding mandated by the content-type MIME type.
361     * <p>
362     * If lenient detection is indicated and the detection above fails as per
363     * specifications it then attempts the following:
364     * <p>
365     * If the content type was 'text/html' it replaces it with 'text/xml' and
366     * tries the detection again.
367     * <p>
368     * Else if the XML prolog had a charset encoding that encoding is used.
369     * <p>
370     * Else if the content type had a charset encoding that encoding is used.
371     * <p>
372     * Else 'UTF-8' is used.
373     * <p>
374     * If lenient detection is indicated an XmlStreamReaderException is never
375     * thrown.
376     *
377     * @param inputStream InputStream to create the reader from.
378     * @param httpContentType content-type header to use for the resolution of
379     *        the charset encoding.
380     * @param lenient indicates if the charset encoding detection should be
381     *        relaxed.
382     * @throws IOException thrown if there is a problem reading the file.
383     * @throws XmlStreamReaderException thrown if the charset encoding could not
384     *         be determined according to the specs.
385     */
386    public XmlStreamReader(final InputStream inputStream, final String httpContentType,
387            final boolean lenient) throws IOException {
388        this(inputStream, httpContentType, lenient, null);
389    }
390
391    /**
392     * Returns the charset encoding of the XmlStreamReader.
393     *
394     * @return charset encoding.
395     */
396    public String getEncoding() {
397        return encoding;
398    }
399
400    /**
401     * Invokes the underlying reader's <code>read(char[], int, int)</code> method.
402     * @param buf the buffer to read the characters into
403     * @param offset The start offset
404     * @param len The number of bytes to read
405     * @return the number of characters read or -1 if the end of stream
406     * @throws IOException if an I/O error occurs
407     */
408    @Override
409    public int read(final char[] buf, final int offset, final int len) throws IOException {
410        return reader.read(buf, offset, len);
411    }
412
413    /**
414     * Closes the XmlStreamReader stream.
415     *
416     * @throws IOException thrown if there was a problem closing the stream.
417     */
418    @Override
419    public void close() throws IOException {
420        reader.close();
421    }
422
423    /**
424     * Process the raw stream.
425     *
426     * @param bom BOMInputStream to detect byte order marks
427     * @param pis BOMInputStream to guess XML encoding
428     * @param lenient indicates if the charset encoding detection should be
429     *        relaxed.
430     * @return the encoding to be used
431     * @throws IOException thrown if there is a problem reading the stream.
432     */
433    private String doRawStream(final BOMInputStream bom, final BOMInputStream pis, final boolean lenient)
434            throws IOException {
435        final String bomEnc      = bom.getBOMCharsetName();
436        final String xmlGuessEnc = pis.getBOMCharsetName();
437        final String xmlEnc = getXmlProlog(pis, xmlGuessEnc);
438        try {
439            return calculateRawEncoding(bomEnc, xmlGuessEnc, xmlEnc);
440        } catch (final XmlStreamReaderException ex) {
441            if (lenient) {
442                return doLenientDetection(null, ex);
443            }
444            throw ex;
445        }
446    }
447
448    /**
449     * Process a HTTP stream.
450     *
451     * @param bom BOMInputStream to detect byte order marks
452     * @param pis BOMInputStream to guess XML encoding
453     * @param httpContentType The HTTP content type
454     * @param lenient indicates if the charset encoding detection should be
455     *        relaxed.
456     * @return the encoding to be used
457     * @throws IOException thrown if there is a problem reading the stream.
458     */
459    private String processHttpStream(final BOMInputStream bom, final BOMInputStream pis, final String httpContentType,
460        final boolean lenient) throws IOException {
461        final String bomEnc = bom.getBOMCharsetName();
462        final String xmlGuessEnc = pis.getBOMCharsetName();
463        final String xmlEnc = getXmlProlog(pis, xmlGuessEnc);
464        try {
465            return calculateHttpEncoding(httpContentType, bomEnc, xmlGuessEnc, xmlEnc, lenient);
466        } catch (final XmlStreamReaderException ex) {
467            if (lenient) {
468                return doLenientDetection(httpContentType, ex);
469            }
470            throw ex;
471        }
472    }
473
474    /**
475     * Do lenient detection.
476     *
477     * @param httpContentType content-type header to use for the resolution of
478     *        the charset encoding.
479     * @param ex The thrown exception
480     * @return the encoding
481     * @throws IOException thrown if there is a problem reading the stream.
482     */
483    private String doLenientDetection(String httpContentType,
484            XmlStreamReaderException ex) throws IOException {
485        if (httpContentType != null && httpContentType.startsWith("text/html")) {
486            httpContentType = httpContentType.substring("text/html".length());
487            httpContentType = "text/xml" + httpContentType;
488            try {
489                return calculateHttpEncoding(httpContentType, ex.getBomEncoding(),
490                        ex.getXmlGuessEncoding(), ex.getXmlEncoding(), true);
491            } catch (final XmlStreamReaderException ex2) {
492                ex = ex2;
493            }
494        }
495        String encoding = ex.getXmlEncoding();
496        if (encoding == null) {
497            encoding = ex.getContentTypeEncoding();
498        }
499        if (encoding == null) {
500            encoding = defaultEncoding == null ? UTF_8 : defaultEncoding;
501        }
502        return encoding;
503    }
504
505    /**
506     * Calculate the raw encoding.
507     *
508     * @param bomEnc BOM encoding
509     * @param xmlGuessEnc XML Guess encoding
510     * @param xmlEnc XML encoding
511     * @return the raw encoding
512     * @throws IOException thrown if there is a problem reading the stream.
513     */
514    String calculateRawEncoding(final String bomEnc, final String xmlGuessEnc,
515            final String xmlEnc) throws IOException {
516
517        // BOM is Null
518        if (bomEnc == null) {
519            if (xmlGuessEnc == null || xmlEnc == null) {
520                return defaultEncoding == null ? UTF_8 : defaultEncoding;
521            }
522            if (xmlEnc.equals(UTF_16) &&
523               (xmlGuessEnc.equals(UTF_16BE) || xmlGuessEnc.equals(UTF_16LE))) {
524                return xmlGuessEnc;
525            }
526            return xmlEnc;
527        }
528
529        // BOM is UTF-8
530        if (bomEnc.equals(UTF_8)) {
531            if (xmlGuessEnc != null && !xmlGuessEnc.equals(UTF_8)) {
532                final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc);
533                throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
534            }
535            if (xmlEnc != null && !xmlEnc.equals(UTF_8)) {
536                final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc);
537                throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
538            }
539            return bomEnc;
540        }
541
542        // BOM is UTF-16BE or UTF-16LE
543        if (bomEnc.equals(UTF_16BE) || bomEnc.equals(UTF_16LE)) {
544            if (xmlGuessEnc != null && !xmlGuessEnc.equals(bomEnc)) {
545                final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc);
546                throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
547            }
548            if (xmlEnc != null && !xmlEnc.equals(UTF_16) && !xmlEnc.equals(bomEnc)) {
549                final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc);
550                throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
551            }
552            return bomEnc;
553        }
554
555        // BOM is UTF-32BE or UTF-32LE
556        if (bomEnc.equals(UTF_32BE) || bomEnc.equals(UTF_32LE)) {
557            if (xmlGuessEnc != null && !xmlGuessEnc.equals(bomEnc)) {
558                final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc);
559                throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
560            }
561            if (xmlEnc != null && !xmlEnc.equals(UTF_32) && !xmlEnc.equals(bomEnc)) {
562                final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc);
563                throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
564            }
565            return bomEnc;
566        }
567
568        // BOM is something else
569        final String msg = MessageFormat.format(RAW_EX_2, bomEnc, xmlGuessEnc, xmlEnc);
570        throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
571    }
572
573
574    /**
575     * Calculate the HTTP encoding.
576     *
577     * @param httpContentType The HTTP content type
578     * @param bomEnc BOM encoding
579     * @param xmlGuessEnc XML Guess encoding
580     * @param xmlEnc XML encoding
581     * @param lenient indicates if the charset encoding detection should be
582     *        relaxed.
583     * @return the HTTP encoding
584     * @throws IOException thrown if there is a problem reading the stream.
585     */
586    String calculateHttpEncoding(final String httpContentType,
587            final String bomEnc, final String xmlGuessEnc, final String xmlEnc,
588            final boolean lenient) throws IOException {
589
590        // Lenient and has XML encoding
591        if (lenient && xmlEnc != null) {
592            return xmlEnc;
593        }
594
595        // Determine mime/encoding content types from HTTP Content Type
596        final String cTMime = getContentTypeMime(httpContentType);
597        final String cTEnc  = getContentTypeEncoding(httpContentType);
598        final boolean appXml  = isAppXml(cTMime);
599        final boolean textXml = isTextXml(cTMime);
600
601        // Mime type NOT "application/xml" or "text/xml"
602        if (!appXml && !textXml) {
603            final String msg = MessageFormat.format(HTTP_EX_3, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
604            throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
605        }
606
607        // No content type encoding
608        if (cTEnc == null) {
609            if (appXml) {
610                return calculateRawEncoding(bomEnc, xmlGuessEnc, xmlEnc);
611            }
612            return defaultEncoding == null ? US_ASCII : defaultEncoding;
613        }
614
615        // UTF-16BE or UTF-16LE content type encoding
616        if (cTEnc.equals(UTF_16BE) || cTEnc.equals(UTF_16LE)) {
617            if (bomEnc != null) {
618                final String msg = MessageFormat.format(HTTP_EX_1, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
619                throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
620            }
621            return cTEnc;
622        }
623
624        // UTF-16 content type encoding
625        if (cTEnc.equals(UTF_16)) {
626            if (bomEnc != null && bomEnc.startsWith(UTF_16)) {
627                return bomEnc;
628            }
629            final String msg = MessageFormat.format(HTTP_EX_2, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
630            throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
631        }
632
633        // UTF-32BE or UTF-132E content type encoding
634        if (cTEnc.equals(UTF_32BE) || cTEnc.equals(UTF_32LE)) {
635            if (bomEnc != null) {
636                final String msg = MessageFormat.format(HTTP_EX_1, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
637                throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
638            }
639            return cTEnc;
640        }
641
642        // UTF-32 content type encoding
643        if (cTEnc.equals(UTF_32)) {
644            if (bomEnc != null && bomEnc.startsWith(UTF_32)) {
645                return bomEnc;
646            }
647            final String msg = MessageFormat.format(HTTP_EX_2, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
648            throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
649        }
650
651        return cTEnc;
652    }
653
654    /**
655     * Returns MIME type or NULL if httpContentType is NULL.
656     *
657     * @param httpContentType the HTTP content type
658     * @return The mime content type
659     */
660    static String getContentTypeMime(final String httpContentType) {
661        String mime = null;
662        if (httpContentType != null) {
663            final int i = httpContentType.indexOf(";");
664            if (i >= 0) {
665                mime = httpContentType.substring(0, i);
666            } else {
667                mime = httpContentType;
668            }
669            mime = mime.trim();
670        }
671        return mime;
672    }
673
674    private static final Pattern CHARSET_PATTERN = Pattern
675            .compile("charset=[\"']?([.[^; \"']]*)[\"']?");
676
677    /**
678     * Returns charset parameter value, NULL if not present, NULL if
679     * httpContentType is NULL.
680     *
681     * @param httpContentType the HTTP content type
682     * @return The content type encoding (upcased)
683     */
684    static String getContentTypeEncoding(final String httpContentType) {
685        String encoding = null;
686        if (httpContentType != null) {
687            final int i = httpContentType.indexOf(";");
688            if (i > -1) {
689                final String postMime = httpContentType.substring(i + 1);
690                final Matcher m = CHARSET_PATTERN.matcher(postMime);
691                encoding = m.find() ? m.group(1) : null;
692                encoding = encoding != null ? encoding.toUpperCase(Locale.ROOT) : null;
693            }
694        }
695        return encoding;
696    }
697
698    /**
699     * Pattern capturing the encoding of the "xml" processing instruction.
700     */
701    public static final Pattern ENCODING_PATTERN = Pattern.compile(
702            "<\\?xml.*encoding[\\s]*=[\\s]*((?:\".[^\"]*\")|(?:'.[^']*'))",
703            Pattern.MULTILINE);
704
705    /**
706     * Returns the encoding declared in the <?xml encoding=...?>, NULL if none.
707     *
708     * @param inputStream InputStream to create the reader from.
709     * @param guessedEnc guessed encoding
710     * @return the encoding declared in the <?xml encoding=...?>
711     * @throws IOException thrown if there is a problem reading the stream.
712     */
713    private static String getXmlProlog(final InputStream inputStream, final String guessedEnc)
714            throws IOException {
715        String encoding = null;
716        if (guessedEnc != null) {
717            final byte[] bytes = new byte[BUFFER_SIZE];
718            inputStream.mark(BUFFER_SIZE);
719            int offset = 0;
720            int max = BUFFER_SIZE;
721            int c = inputStream.read(bytes, offset, max);
722            int firstGT = -1;
723            String xmlProlog = ""; // avoid possible NPE warning (cannot happen; this just silences the warning)
724            while (c != -1 && firstGT == -1 && offset < BUFFER_SIZE) {
725                offset += c;
726                max -= c;
727                c = inputStream.read(bytes, offset, max);
728                xmlProlog = new String(bytes, 0, offset, guessedEnc);
729                firstGT = xmlProlog.indexOf('>');
730            }
731            if (firstGT == -1) {
732                if (c == -1) {
733                    throw new IOException("Unexpected end of XML stream");
734                }
735                throw new IOException(
736                        "XML prolog or ROOT element not found on first "
737                                + offset + " bytes");
738            }
739            final int bytesRead = offset;
740            if (bytesRead > 0) {
741                inputStream.reset();
742                final BufferedReader bReader = new BufferedReader(new StringReader(
743                        xmlProlog.substring(0, firstGT + 1)));
744                final StringBuffer prolog = new StringBuffer();
745                String line;
746                while ((line = bReader.readLine()) != null) {
747                    prolog.append(line);
748                }
749                final Matcher m = ENCODING_PATTERN.matcher(prolog);
750                if (m.find()) {
751                    encoding = m.group(1).toUpperCase(Locale.ROOT);
752                    encoding = encoding.substring(1, encoding.length() - 1);
753                }
754            }
755        }
756        return encoding;
757    }
758
759    /**
760     * Indicates if the MIME type belongs to the APPLICATION XML family.
761     *
762     * @param mime The mime type
763     * @return true if the mime type belongs to the APPLICATION XML family,
764     * otherwise false
765     */
766    static boolean isAppXml(final String mime) {
767        return mime != null &&
768               (mime.equals("application/xml") ||
769                mime.equals("application/xml-dtd") ||
770                mime.equals("application/xml-external-parsed-entity") ||
771               mime.startsWith("application/") && mime.endsWith("+xml"));
772    }
773
774    /**
775     * Indicates if the MIME type belongs to the TEXT XML family.
776     *
777     * @param mime The mime type
778     * @return true if the mime type belongs to the TEXT XML family,
779     * otherwise false
780     */
781    static boolean isTextXml(final String mime) {
782        return mime != null &&
783              (mime.equals("text/xml") ||
784               mime.equals("text/xml-external-parsed-entity") ||
785              mime.startsWith("text/") && mime.endsWith("+xml"));
786    }
787
788    private static final String RAW_EX_1 =
789        "Invalid encoding, BOM [{0}] XML guess [{1}] XML prolog [{2}] encoding mismatch";
790
791    private static final String RAW_EX_2 =
792        "Invalid encoding, BOM [{0}] XML guess [{1}] XML prolog [{2}] unknown BOM";
793
794    private static final String HTTP_EX_1 =
795        "Invalid encoding, CT-MIME [{0}] CT-Enc [{1}] BOM [{2}] XML guess [{3}] XML prolog [{4}], BOM must be NULL";
796
797    private static final String HTTP_EX_2 =
798        "Invalid encoding, CT-MIME [{0}] CT-Enc [{1}] BOM [{2}] XML guess [{3}] XML prolog [{4}], encoding mismatch";
799
800    private static final String HTTP_EX_3 =
801        "Invalid encoding, CT-MIME [{0}] CT-Enc [{1}] BOM [{2}] XML guess [{3}] XML prolog [{4}], Invalid MIME";
802
803}