TS Bartlett <###@###.###>

After setting <Space> to be a word character, io.StreamTokenizer will not allow a
space or spaces alone to form a word token.

Steps to reproduce
Save attached text to TokenSpaces.txt
Compile and run attached code
// Note: the last line of the input file should have <Tab><Space><Space><Tab> in it,
// and the StreamTokenizer should keep the two spaces as a word token

------ Input file -----
This is a test of token spaces, this line should be one word.
This line should be all separate tokens.
This line should have two tokens.
This line should have three tokens.
This line should have a space word token.

----- Source File -----
// sting and stream tokenizer tests
import java.io.StreamTokenizer;
import java.io.FileInputStream;
import java.io.IOException;

class TokenSpaceTest
{
   public static void main(String argv[])
   {
      // local variables
      String sType;
      try
      {
         StreamTokenizer st = new StreamTokenizer( new FileInputStream( "TokenSpaces.txt") );
         st.eolIsSignificant( true );
         st.wordChars( 32, 32 );

         while ( st.nextToken() != StreamTokenizer.TT_EOF )
         {
            switch (st.ttype)
            {
               case StreamTokenizer.TT_EOL:
                  sType = "EOL";
                  break;
               case StreamTokenizer.TT_EOF:
                  sType = "EOF";
                  break;
               case StreamTokenizer.TT_NUMBER:
                  sType = "Number";
                  break;
               case StreamTokenizer.TT_WORD:
                  sType = "Word";
                  break;
               default:
                  sType = "Unknown";
                  break;

            } // endSwitch:

            System.out.println( "TokenType: " + sType + ", Value: " + st.sval );
         }
      }
      catch ( IOException e ) {}
      
   }

}