Fix BOM / encoding problems

* enable testhst_lhs_007, testhst_lhs_008 and testhst_lhs_009 for InputStream
* disable those tests on readers, as readers bypass any encoding
* do not try to discover the encoding used when the input is given a Reader
* add an SIO-8859-1 encoded coment in the test xml (testEncodingISO_8859_1_newReader and testEncodingISO_8859_1_InputStream_encoded tests do decode it wrongly as they use UTF-8)

Author: gnodet

src/main/java/org/codehaus/plexus/util/xml/XmlReader.java

@@ -504,11 +504,8 @@ else if ( bomEnc.equals( UTF_8 ) )
         }
         else if ( bomEnc.equals( UTF_16BE ) || bomEnc.equals( UTF_16LE ) )
         {
-            if ( xmlGuessEnc != null && !xmlGuessEnc.equals( bomEnc ) )
-            {
-                throw new IOException( RAW_EX_1.format( new Object[] { bomEnc, xmlGuessEnc, xmlEnc } ) );
-            }
-            if ( xmlEnc != null && !xmlEnc.equals( UTF_16 ) && !xmlEnc.equals( bomEnc ) )
+            if ( xmlGuessEnc != null && !xmlGuessEnc.equals( bomEnc )
+                || xmlEnc != null && !xmlEnc.equals( UTF_16 ) && !xmlEnc.equals( bomEnc ) )
             {
                 throw new XmlStreamReaderException( RAW_EX_1.format( new Object[] { bomEnc, xmlGuessEnc, xmlEnc } ),
                                                     bomEnc, xmlGuessEnc, xmlEnc, is );

src/main/java/org/codehaus/plexus/util/xml/pull/MXParser.java

@@ -11,10 +11,13 @@
 
 import java.io.EOFException;
 import java.io.IOException;
+import java.io.InputStreamReader;
 import java.io.Reader;
 import java.io.UnsupportedEncodingException;
 
-import org.codehaus.plexus.util.ReaderFactory;
+import org.codehaus.plexus.util.xml.XmlReader;
+import org.codehaus.plexus.util.xml.XmlStreamReader;
+import org.codehaus.plexus.util.xml.XmlStreamReaderException;
 
 //import java.util.Hashtable;
 
@@ -122,6 +125,7 @@ private String newStringIntern( char[] cbuf, int off, int len )
     // private String elValue[];
     private int elNamespaceCount[];
 
+    private String fileEncoding = null;
 
     /**
      * Make sure that we have enough space to keep element stack if passed size. It will always create one additional
@@ -675,18 +679,30 @@ public void setInput( java.io.InputStream inputStream, String inputEncoding )
         {
             if ( inputEncoding != null )
             {
-                reader = ReaderFactory.newReader( inputStream, inputEncoding );
+                reader = new InputStreamReader( inputStream, inputEncoding );
             }
             else
             {
-                reader = ReaderFactory.newXmlReader( inputStream );
+                reader = new XmlStreamReader( inputStream, false );
             }
         }
         catch ( UnsupportedEncodingException une )
         {
             throw new XmlPullParserException( "could not create reader for encoding " + inputEncoding + " : " + une,
                                               this, une );
         }
+        catch ( XmlStreamReaderException e )
+        {
+            if ( "UTF-8".equals( e.getBomEncoding() ) )
+            {
+                throw new XmlPullParserException( "UTF-8 BOM plus xml decl of " + e.getXmlEncoding() + " is incompatible", this, e );
+            }
+            if ( e.getBomEncoding() != null && e.getBomEncoding().startsWith( "UTF-16" ) )
+            {
+                throw new XmlPullParserException( "UTF-16 BOM in a " + e.getXmlEncoding() + " encoded file is incompatible", this, e );
+            }
+            throw new XmlPullParserException( "could not create reader : " + e, this, e );
+        }
         catch ( IOException e )
         {
             throw new XmlPullParserException( "could not create reader : " + e, this, e );
@@ -3415,7 +3431,7 @@ private void parseXmlDeclWithVersion( int versionStart, int versionEnd )
             final int encodingEnd = pos - 1;
 
             // TODO reconcile with setInput encodingName
-            // inputEncoding = newString( buf, encodingStart, encodingEnd - encodingStart );
+            inputEncoding = newString( buf, encodingStart, encodingEnd - encodingStart );
 
             lastParsedAttr = "encoding";
 

src/test/java/org/codehaus/plexus/util/xml/pull/eduni_misc_Test_BjoernHoehrmannviaHST2013_09_18_Test.java

@@ -4,8 +4,10 @@
 import static org.junit.Assert.fail;
 
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.FileReader;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.Reader;
 import java.nio.charset.StandardCharsets;
 
@@ -206,24 +208,21 @@ public void testhst_bh_006()
      * Version:
      *
      * @throws java.io.IOException if there is an I/O error
-     *
-     * NOTE: This test is SKIPPED as the MXParser object alone is unable to detect whether UTF-8 file
-     * has a BOM or not
      */
-    // @Test
+    @Test
     public void testhst_lhs_007()
         throws IOException
     {
-        try ( Reader reader = ReaderFactory.newXmlReader( new File( testResourcesDir, "007.xml" ) ) )
+        try ( InputStream is = new FileInputStream( new File( testResourcesDir, "007.xml" ) ) )
         {
-            parser.setInput( reader );
+            parser.setInput( is, null );
             while ( parser.nextToken() != XmlPullParser.END_DOCUMENT )
                 ;
-            fail( "UTF-8 BOM plus xml decl of iso-8859-1 incompatible" );
+            fail( "UTF-8 BOM plus xml decl of ISO-8859-1 incompatible" );
         }
         catch ( XmlPullParserException e )
         {
-            assertTrue( e.getMessage().contains( "UTF-8 BOM plus xml decl of iso-8859-1 is incompatible" ) );
+            assertTrue( e.getMessage().contains( "UTF-8 BOM plus xml decl of ISO-8859-1 is incompatible" ) );
         }
     }
 
@@ -235,57 +234,24 @@ public void testhst_lhs_007()
      * Version:
      *
      * @throws java.io.IOException if there is an I/O error
-     *
-     * NOTE: This test is SKIPPED as the MXParser object alone is unable to detect whether UTF-16 file
-     * has a BOM or not
      */
-    // @Test
-    public void testhst_lhs_008_newReader()
+    @Test
+    public void testhst_lhs_008()
         throws IOException
     {
-        try ( Reader reader =
-            ReaderFactory.newReader( new File( testResourcesDir, "008.xml" ), StandardCharsets.UTF_16.name() ) )
+        try ( InputStream is = new FileInputStream( new File( testResourcesDir, "008.xml" ) ) )
         {
-            parser.setInput( reader );
+            parser.setInput( is, null );
             while ( parser.nextToken() != XmlPullParser.END_DOCUMENT )
                 ;
-            fail( "UTF-16 BOM plus xml decl of utf-8 (using UTF-16 coding) incompatible" );
+            fail( "UTF-16 BOM plus xml decl of UTF-8 (using UTF-16 coding) incompatible" );
         }
         catch ( XmlPullParserException e )
         {
             assertTrue( e.getMessage().contains( "UTF-16 BOM in a UTF-8 encoded file is incompatible" ) );
         }
     }
 
-    /**
-     * Test ID: <pre>hst-lhs-008</pre>
-     * Test URI: <pre>008.xml</pre>
-     * Comment: <pre>UTF-16 BOM plus xml decl of utf-8 (using UTF-16 coding) incompatible</pre>
-     * Sections: <pre>4.3.3</pre>
-     * Version:
-     *
-     * @throws java.io.IOException if there is an I/O error
-     *
-     * NOTE: This test is SKIPPED as MXParser is unable to detect UTF-16 BOM detection when chars are read as
-     * UTF-8, and XmlReader in lenient mode does not throw exception.
-     */
-    // @Test
-    public void testhst_lhs_008_XmlReader()
-        throws IOException
-    {
-        try ( Reader reader = ReaderFactory.newXmlReader( new File( testResourcesDir, "008.xml" ) ) )
-        {
-            parser.setInput( reader );
-            while ( parser.nextToken() != XmlPullParser.END_DOCUMENT )
-                ;
-            fail( "UTF-16 BOM plus xml decl of utf-8 (using UTF-16 coding) incompatible" );
-        }
-        catch ( XmlPullParserException e )
-        {
-            assertTrue( e.getMessage().contains( "UTF-16 BOM plus xml decl of utf-8 is incompatible" ) );
-        }
-    }
-
     /**
      * Test ID: <pre>hst-lhs-009</pre>
      * Test URI: <pre>009.xml</pre>
@@ -298,52 +264,20 @@ public void testhst_lhs_008_XmlReader()
      * NOTE: This test is SKIPPED as MXParser is unable to detect UTF-16 BOM detection when chars are read as
      * UTF-8.
      */
-    // @Test
-    public void testhst_lhs_009_newReader()
-        throws IOException
-    {
-        try ( Reader reader =
-            ReaderFactory.newReader( new File( testResourcesDir, "009.xml" ), StandardCharsets.UTF_16.name() ) )
-        {
-            parser.setInput( reader );
-            while ( parser.nextToken() != XmlPullParser.END_DOCUMENT )
-                ;
-            fail( "UTF-16 BOM plus xml decl of utf-8 (using UTF-8 coding) incompatible" );
-        }
-        catch ( XmlPullParserException e )
-        {
-            assertTrue( e.getMessage().contains( "UTF-16 BOM in a UTF-8 encoded file is incompatible" ) );
-        }
-    }
-
-    /**
-     * Test ID: <pre>hst-lhs-009</pre>
-     * Test URI: <pre>009.xml</pre>
-     * Comment: <pre>UTF-16 BOM plus xml decl of utf-8 (using UTF-8 coding) incompatible</pre>
-     * Sections: <pre>4.3.3</pre>
-     * Version:
-     *
-     * @throws java.io.IOException if there is an I/O error
-     */
     @Test
-    public void testhst_lhs_009_XmlReader()
+    public void testhst_lhs_009()
         throws IOException
     {
-        try ( Reader reader = ReaderFactory.newXmlReader( new File( testResourcesDir, "009.xml" ) ) )
+        try ( InputStream is = new FileInputStream( new File( testResourcesDir, "009.xml" ) ) )
         {
-            parser.setInput( reader );
+            parser.setInput( is, null );
             while ( parser.nextToken() != XmlPullParser.END_DOCUMENT )
                 ;
-            fail( "UTF-16 BOM plus xml decl of utf-8 (using UTF-8 coding) incompatible" );
-        }
-        catch ( IOException e )
-        {
-            // even when XmlReader is in lenient mode, it throws an IOException
-            assertTrue( e.getMessage().contains( "Invalid encoding, BOM [UTF-16BE] XML guess [UTF-8] XML prolog [UTF-8] encoding mismatch" ) );
+            fail( "UTF-16 BOM plus xml decl of UTF-8 (using UTF-8 coding) incompatible" );
         }
         catch ( XmlPullParserException e )
         {
-            fail( "Encoding problem should be detected by the XmlReader" );
+            assertTrue( e.getMessage(), e.getMessage().contains( "UTF-16 BOM in a UTF-8 encoded file is incompatible" ) );
         }
     }
 

src/test/resources/xml/test-encoding-ISO-8859-1.xml

@@ -1,2 +1,3 @@
 <?xml version="1.0" encoding="ISO-8859-1"?>
+<!-- Æ -->
 <mytag/>