FULL PRODUCT VERSION :
java version "1.5.0_06"
Java(TM) 2 Runtime Environment, Standard Edition (build 1.5.0_06-b05)
Java HotSpot(TM) Client VM (build 1.5.0_06-b05, mixed mode, sharing)
ADDITIONAL OS VERSION INFORMATION :
Microsoft Windows XP [Version 5.1.2600]
EXTRA RELEVANT SYSTEM CONFIGURATION :
Similar (even worse) performance on Mac OS X 10.4.5 running Java 5 Release 3
A DESCRIPTION OF THE PROBLEM :
In evaluating and exploring the NIO packages I discovered that the performance of ByteBuffer is extremly low (up to 10 to 15 times slower) if the size is not equal to a power of two.
ByteBuffers obtained through a call to map() seem not to be affected.
I only used ByteBuffers to read files. Reading from sockets may or may not be affected.
STEPS TO FOLLOW TO REPRODUCE THE PROBLEM :
Allocate one ByteBuffer with a size of 16384 bytes and another with a size of 16383 bytes. Read a file with both buffers and compare the execution time.
EXPECTED VERSUS ACTUAL BEHAVIOR :
EXPECTED -
I expected only little to none impact of minimal buffer size differences on performance.
ACTUAL -
ByteBuffers with sizes equal to powers of two (8192, 16384, etc.) are 10 to 15 times faster than "unaligned" buffers.
This is a sample output from my program:
Runs per bufsize: 1000
Reading file: "test15054.tmp", size: 1048576
Testing buffer size: 16384
NIO allocate: 2213 ms
NIO wrap: 2143 ms
NIO allocateDirect: 1552 ms
NIO map: 1031 ms
IO: 1953 ms
Testing buffer size: 16383
NIO allocate: 33966 ms
NIO wrap: 34358 ms
NIO allocateDirect: 28561 ms
NIO map: 1121 ms
IO: 2183 ms
Testing buffer size: 16385
NIO allocate: 10415 ms
NIO wrap: 10355 ms
NIO allocateDirect: 8391 ms
NIO map: 1262 ms
IO: 2233 ms
Testing buffer size: 16000
NIO allocate: 10325 ms
NIO wrap: 10364 ms
NIO allocateDirect: 8352 ms
NIO map: 1262 ms
IO: 2353 ms
Finished
REPRODUCIBILITY :
This bug can be reproduced always.
---------- BEGIN SOURCE ----------
import java.io.BufferedOutputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.util.Random;
public class TestNio
{
private static final long FILESIZE = 1024 * 1024 * 1;
private static final int RUNS = 1000;
private static final int BUFSIZE_BASE = (1 << 14);
private static final int[] BUFSIZE_ARRAY = new int[] { BUFSIZE_BASE,
BUFSIZE_BASE - 1,
BUFSIZE_BASE + 1,
BUFSIZE_BASE / 1000 * 1000 };
public static void main(String[] args)
{
try {
System.out.println("Runs per bufsize: " + RUNS);
final File file = createTestFile();
System.out.println("Reading file: \""+ file.getName()+ "\", size: "+ file.length());
for (int i = 0; i < BUFSIZE_ARRAY.length; ++i) {
final int bufsize = BUFSIZE_ARRAY[i];
System.out.println("");
System.out.println("Testing buffer size: " + bufsize);
ByteArrayOutputStream baos = new ByteArrayOutputStream(
(int)file.length());
testNewIOAllocate(file, baos, bufsize);
testNewIOWrap(file, baos, bufsize);
testNewIOAllocateDirect(file, baos, bufsize);
testNewIOMap(file, baos, bufsize);
testOldIO(file, baos, bufsize);
}
System.out.println("Finished");
} catch (IOException e) {
e.printStackTrace();
}
}
private static File createTestFile() throws IOException
{
final File file = File.createTempFile("test", null);
file.deleteOnExit();
{
final Random random = new Random();
{
OutputStream output = new BufferedOutputStream(
new FileOutputStream(file));
try {
for (long i = 0; i < FILESIZE; ++i) {
output.write(random.nextInt(256));
}
} finally {
output.close();
}
}
}
return file;
}
private static void testNewIOMap(final File file,
final ByteArrayOutputStream baos, final int bufsize)
throws FileNotFoundException, IOException
{
System.out.print("NIO map: ");
final long t1 = System.currentTimeMillis();
FileInputStream input = new FileInputStream(file);
FileChannel inputChannel = input.getChannel();
try {
MappedByteBuffer inputBuffer = inputChannel.map(FileChannel.MapMode.READ_ONLY, 0, inputChannel.size());
byte[] buf = new byte[bufsize];
int len;
for (int i = 0; i < RUNS; ++i) {
baos.reset();
inputBuffer.position(0);
long c = 0;
while ((len = Math.min(buf.length, inputBuffer.remaining())) > 0) {
inputBuffer.get(buf, 0, len);
baos.write(buf, 0, len);
c += len;
}
if (c != inputChannel.size()) {
throw new IllegalStateException(c + " != " + len);
}
}
}finally{
inputChannel.close();
input.close();
}
final long t2 = System.currentTimeMillis();
System.out.println((t2 - t1) + " ms");
}
private static void testNewIOAllocateDirect(final File file,
final ByteArrayOutputStream baos, final int bufsize)
throws FileNotFoundException, IOException
{
System.out.print("NIO allocateDirect: ");
final long t1 = System.currentTimeMillis();
FileInputStream input = new FileInputStream(file);
FileChannel inputChannel = input.getChannel();
try {
ByteBuffer inputBuffer = ByteBuffer.allocateDirect(bufsize);
byte[] buf = new byte[bufsize];
int len;
for (int i = 0; i < RUNS; ++i) {
baos.reset();
inputChannel.position(0);
long c = 0;
while ((len = inputChannel.read(inputBuffer)) >= 0) {
inputBuffer.flip();
inputBuffer.get(buf, 0, len);
baos.write(buf, 0, len);
inputBuffer.flip();
c += len;
}
if (c != inputChannel.size()) {
throw new IllegalStateException(c + " != " + len);
}
}
}finally{
inputChannel.close();
input.close();
}
final long t2 = System.currentTimeMillis();
System.out.println((t2 - t1) + " ms");
}
private static void testNewIOAllocate(final File file,
final ByteArrayOutputStream baos, final int bufsize)
throws FileNotFoundException, IOException
{
System.out.print("NIO allocate: ");
final long t1 = System.currentTimeMillis();
FileInputStream input = new FileInputStream(file);
FileChannel inputChannel = input.getChannel();
try {
ByteBuffer inputBuffer = ByteBuffer.allocate(bufsize);
byte[] buf = new byte[bufsize];
int len;
for (int i = 0; i < RUNS; ++i) {
baos.reset();
inputChannel.position(0);
long c = 0;
while ((len = inputChannel.read(inputBuffer)) >= 0) {
inputBuffer.flip();
inputBuffer.get(buf, 0, len);
baos.write(buf, 0, len);
inputBuffer.flip();
c += len;
}
if (c != inputChannel.size()) {
throw new IllegalStateException(c + " != " + len);
}
}
}finally{
inputChannel.close();
input.close();
}
final long t2 = System.currentTimeMillis();
System.out.println((t2 - t1) + " ms");
}
private static void testNewIOWrap(final File file, final ByteArrayOutputStream baos, final int bufsize)
throws FileNotFoundException, IOException
{
System.out.print("NIO wrap: ");
final long t1 = System.currentTimeMillis();
FileInputStream input = new FileInputStream(file);
FileChannel inputChannel = input.getChannel();
try {
byte[] wrapBuf = new byte[bufsize];
ByteBuffer inputBuffer = ByteBuffer.wrap(wrapBuf);
byte[] buf = new byte[bufsize];
int len;
for (int i = 0; i < RUNS; ++i) {
baos.reset();
inputChannel.position(0);
long c = 0;
while ((len = inputChannel.read(inputBuffer)) >= 0) {
inputBuffer.flip();
inputBuffer.get(buf, 0, len);
baos.write(buf, 0, len);
inputBuffer.flip();
c += len;
}
if (c != inputChannel.size()) {
throw new IllegalStateException(c + " != " + len);
}
}
}finally{
inputChannel.close();
input.close();
}
final long t2 = System.currentTimeMillis();
System.out.println((t2 - t1) + " ms");
}
private static void testOldIO(final File file, final ByteArrayOutputStream baos, final int bufsize)
throws FileNotFoundException, IOException
{
System.out.print("IO: ");
final long t1 = System.currentTimeMillis();
byte[] buf = new byte[bufsize];
int len;
for (int i = 0; i < RUNS; ++i) {
FileInputStream input = new FileInputStream(file);
try {
baos.reset();
while ((len = input.read(buf)) >= 0) {
baos.write(buf, 0, len);
}
}finally{
input.close();
}
}
final long t2 = System.currentTimeMillis();
System.out.println((t2 - t1) + " ms");
}
}
---------- END SOURCE ----------
CUSTOMER SUBMITTED WORKAROUND :
Use allocated buffers with size of 8192, 16384, etc. Or use mapped buffers if reading from files.
java version "1.5.0_06"
Java(TM) 2 Runtime Environment, Standard Edition (build 1.5.0_06-b05)
Java HotSpot(TM) Client VM (build 1.5.0_06-b05, mixed mode, sharing)
ADDITIONAL OS VERSION INFORMATION :
Microsoft Windows XP [Version 5.1.2600]
EXTRA RELEVANT SYSTEM CONFIGURATION :
Similar (even worse) performance on Mac OS X 10.4.5 running Java 5 Release 3
A DESCRIPTION OF THE PROBLEM :
In evaluating and exploring the NIO packages I discovered that the performance of ByteBuffer is extremly low (up to 10 to 15 times slower) if the size is not equal to a power of two.
ByteBuffers obtained through a call to map() seem not to be affected.
I only used ByteBuffers to read files. Reading from sockets may or may not be affected.
STEPS TO FOLLOW TO REPRODUCE THE PROBLEM :
Allocate one ByteBuffer with a size of 16384 bytes and another with a size of 16383 bytes. Read a file with both buffers and compare the execution time.
EXPECTED VERSUS ACTUAL BEHAVIOR :
EXPECTED -
I expected only little to none impact of minimal buffer size differences on performance.
ACTUAL -
ByteBuffers with sizes equal to powers of two (8192, 16384, etc.) are 10 to 15 times faster than "unaligned" buffers.
This is a sample output from my program:
Runs per bufsize: 1000
Reading file: "test15054.tmp", size: 1048576
Testing buffer size: 16384
NIO allocate: 2213 ms
NIO wrap: 2143 ms
NIO allocateDirect: 1552 ms
NIO map: 1031 ms
IO: 1953 ms
Testing buffer size: 16383
NIO allocate: 33966 ms
NIO wrap: 34358 ms
NIO allocateDirect: 28561 ms
NIO map: 1121 ms
IO: 2183 ms
Testing buffer size: 16385
NIO allocate: 10415 ms
NIO wrap: 10355 ms
NIO allocateDirect: 8391 ms
NIO map: 1262 ms
IO: 2233 ms
Testing buffer size: 16000
NIO allocate: 10325 ms
NIO wrap: 10364 ms
NIO allocateDirect: 8352 ms
NIO map: 1262 ms
IO: 2353 ms
Finished
REPRODUCIBILITY :
This bug can be reproduced always.
---------- BEGIN SOURCE ----------
import java.io.BufferedOutputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.util.Random;
public class TestNio
{
private static final long FILESIZE = 1024 * 1024 * 1;
private static final int RUNS = 1000;
private static final int BUFSIZE_BASE = (1 << 14);
private static final int[] BUFSIZE_ARRAY = new int[] { BUFSIZE_BASE,
BUFSIZE_BASE - 1,
BUFSIZE_BASE + 1,
BUFSIZE_BASE / 1000 * 1000 };
public static void main(String[] args)
{
try {
System.out.println("Runs per bufsize: " + RUNS);
final File file = createTestFile();
System.out.println("Reading file: \""+ file.getName()+ "\", size: "+ file.length());
for (int i = 0; i < BUFSIZE_ARRAY.length; ++i) {
final int bufsize = BUFSIZE_ARRAY[i];
System.out.println("");
System.out.println("Testing buffer size: " + bufsize);
ByteArrayOutputStream baos = new ByteArrayOutputStream(
(int)file.length());
testNewIOAllocate(file, baos, bufsize);
testNewIOWrap(file, baos, bufsize);
testNewIOAllocateDirect(file, baos, bufsize);
testNewIOMap(file, baos, bufsize);
testOldIO(file, baos, bufsize);
}
System.out.println("Finished");
} catch (IOException e) {
e.printStackTrace();
}
}
private static File createTestFile() throws IOException
{
final File file = File.createTempFile("test", null);
file.deleteOnExit();
{
final Random random = new Random();
{
OutputStream output = new BufferedOutputStream(
new FileOutputStream(file));
try {
for (long i = 0; i < FILESIZE; ++i) {
output.write(random.nextInt(256));
}
} finally {
output.close();
}
}
}
return file;
}
private static void testNewIOMap(final File file,
final ByteArrayOutputStream baos, final int bufsize)
throws FileNotFoundException, IOException
{
System.out.print("NIO map: ");
final long t1 = System.currentTimeMillis();
FileInputStream input = new FileInputStream(file);
FileChannel inputChannel = input.getChannel();
try {
MappedByteBuffer inputBuffer = inputChannel.map(FileChannel.MapMode.READ_ONLY, 0, inputChannel.size());
byte[] buf = new byte[bufsize];
int len;
for (int i = 0; i < RUNS; ++i) {
baos.reset();
inputBuffer.position(0);
long c = 0;
while ((len = Math.min(buf.length, inputBuffer.remaining())) > 0) {
inputBuffer.get(buf, 0, len);
baos.write(buf, 0, len);
c += len;
}
if (c != inputChannel.size()) {
throw new IllegalStateException(c + " != " + len);
}
}
}finally{
inputChannel.close();
input.close();
}
final long t2 = System.currentTimeMillis();
System.out.println((t2 - t1) + " ms");
}
private static void testNewIOAllocateDirect(final File file,
final ByteArrayOutputStream baos, final int bufsize)
throws FileNotFoundException, IOException
{
System.out.print("NIO allocateDirect: ");
final long t1 = System.currentTimeMillis();
FileInputStream input = new FileInputStream(file);
FileChannel inputChannel = input.getChannel();
try {
ByteBuffer inputBuffer = ByteBuffer.allocateDirect(bufsize);
byte[] buf = new byte[bufsize];
int len;
for (int i = 0; i < RUNS; ++i) {
baos.reset();
inputChannel.position(0);
long c = 0;
while ((len = inputChannel.read(inputBuffer)) >= 0) {
inputBuffer.flip();
inputBuffer.get(buf, 0, len);
baos.write(buf, 0, len);
inputBuffer.flip();
c += len;
}
if (c != inputChannel.size()) {
throw new IllegalStateException(c + " != " + len);
}
}
}finally{
inputChannel.close();
input.close();
}
final long t2 = System.currentTimeMillis();
System.out.println((t2 - t1) + " ms");
}
private static void testNewIOAllocate(final File file,
final ByteArrayOutputStream baos, final int bufsize)
throws FileNotFoundException, IOException
{
System.out.print("NIO allocate: ");
final long t1 = System.currentTimeMillis();
FileInputStream input = new FileInputStream(file);
FileChannel inputChannel = input.getChannel();
try {
ByteBuffer inputBuffer = ByteBuffer.allocate(bufsize);
byte[] buf = new byte[bufsize];
int len;
for (int i = 0; i < RUNS; ++i) {
baos.reset();
inputChannel.position(0);
long c = 0;
while ((len = inputChannel.read(inputBuffer)) >= 0) {
inputBuffer.flip();
inputBuffer.get(buf, 0, len);
baos.write(buf, 0, len);
inputBuffer.flip();
c += len;
}
if (c != inputChannel.size()) {
throw new IllegalStateException(c + " != " + len);
}
}
}finally{
inputChannel.close();
input.close();
}
final long t2 = System.currentTimeMillis();
System.out.println((t2 - t1) + " ms");
}
private static void testNewIOWrap(final File file, final ByteArrayOutputStream baos, final int bufsize)
throws FileNotFoundException, IOException
{
System.out.print("NIO wrap: ");
final long t1 = System.currentTimeMillis();
FileInputStream input = new FileInputStream(file);
FileChannel inputChannel = input.getChannel();
try {
byte[] wrapBuf = new byte[bufsize];
ByteBuffer inputBuffer = ByteBuffer.wrap(wrapBuf);
byte[] buf = new byte[bufsize];
int len;
for (int i = 0; i < RUNS; ++i) {
baos.reset();
inputChannel.position(0);
long c = 0;
while ((len = inputChannel.read(inputBuffer)) >= 0) {
inputBuffer.flip();
inputBuffer.get(buf, 0, len);
baos.write(buf, 0, len);
inputBuffer.flip();
c += len;
}
if (c != inputChannel.size()) {
throw new IllegalStateException(c + " != " + len);
}
}
}finally{
inputChannel.close();
input.close();
}
final long t2 = System.currentTimeMillis();
System.out.println((t2 - t1) + " ms");
}
private static void testOldIO(final File file, final ByteArrayOutputStream baos, final int bufsize)
throws FileNotFoundException, IOException
{
System.out.print("IO: ");
final long t1 = System.currentTimeMillis();
byte[] buf = new byte[bufsize];
int len;
for (int i = 0; i < RUNS; ++i) {
FileInputStream input = new FileInputStream(file);
try {
baos.reset();
while ((len = input.read(buf)) >= 0) {
baos.write(buf, 0, len);
}
}finally{
input.close();
}
}
final long t2 = System.currentTimeMillis();
System.out.println((t2 - t1) + " ms");
}
}
---------- END SOURCE ----------
CUSTOMER SUBMITTED WORKAROUND :
Use allocated buffers with size of 8192, 16384, etc. Or use mapped buffers if reading from files.