Im restarting this thread (we tried to open this case as of November last year) since we still consider this to be a JDK interoperability problem. I would like to give you the findings we have seen so far with trying to interoperate with NFS hosted on HP-UX. The main issue is with the default method implementation of TryLock() defines a file lock size of long.MAX_VALUE instead of 0):

Using a repro (attached) we found that on HP-UX systems running VxFS
(JFS) the maximal file size is 2 TB. Hence, locking the file using the long.MAX_VALUE (which is a size larger than 2 TB) triggers a local error on the NFS-server which is then propagated to the client.

We saw the same result using both Solaris and Linux clients on an NFS
file system exported from a HP-UX machine (HP-UX 11.23). We had no
problems locking a file using the long.MAX_VALUE using both Solaris
and Linux clients when the NFS system was exported from a Solaris
machine.

As far as I understand it, the purpose of using long.MAX_VALUE is to
ensure that the whole file will always be locked independent of what
size it might grow to in the future. To accomplish this we can use the value 0 to indicate this behaviour.

The Solaris 10, and Linux 2.6 fcntl(2) man pages says:

'A lock will be set to extend to the largest possible value of the
file offset for that file by setting l_len to 0. If such a lock also
has l_start set to 0 and l_whence is set to SEEK_SET, the whole file
will be locked.'

The HP-UX 11i version 2 page says

'A lock can be set to always extend to the end of file by setting
l_len to zero (0). If such a lock also has l_start set to zero (0),
the whole file will be locked.'

I interpret this as the conversion to the maximal value of the length
is performed after the actual system call on all platforms. For me it
makes sense to try to retain the 0 length magical value as far as
possible in the chain and let the limitations of the host OS determine how to lock a whole file. By explicitly converting the value to long.MAX_VALUE in the JDK (which is the case today) we might
encounter platform restrictions when this value is propagated to a
host system not supporting full 64-bit file sizes.

This is the repro used:
---------------------------------------------------------------------
#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <string.h>
#include <errno.h>
#include <stdint.h>

static void usage() {
  fprintf(stderr,"Specify the file to test using flag -f\n");
  fprintf(stderr,"Specify the byte length of the lock using the flag -l\n");
  fprintf(stderr,"A negative length will use INT64_MAX from stdint.h\n");
}

static int lock_file(int fd, long len);

struct flock lock_data;

int main(int argc, char *argv[], char *envp[]) {
  int fd, i;
  int status;
  long buf;
  char *filename;
  off_t len;
  extern char *optarg;
  extern int optind, optopt;

  if(argc < 2) {
    usage();
    exit(EXIT_FAILURE);
  }

  printf("Sizeof off_t is %d\n",sizeof(off_t));
  printf("Sizeof long is %d\n",sizeof(long));
  printf("Sizeof struct flock is %d\n",sizeof(struct flock));

  len = 0;

  while ((status = getopt(argc, argv, "f:l:")) != -1) {
    switch(status) {
    case 'f':
      filename = optarg;
      break;
    case 'l':
      buf = atol(optarg);
      if( buf < 0 )
len = 0;
      else
len = buf;
      break;
    case ':':
      fprintf(stderr,"Option -%c requires a filename\n",optopt);
      exit(EXIT_FAILURE);
      break;
    case '?':
      fprintf(stderr,"Unrecognized option: -%c\n",optopt);
      usage();
      exit(EXIT_FAILURE);
    }
  }
  
  fd = open(filename, O_RDWR, S_IRUSR | S_IWUSR );
  if( fd == -1 ) {
    fprintf(stderr,"Cannot open file %s\n",filename);
    perror("open");
    exit(EXIT_FAILURE);
  }

  /* Init the flock */
  status = fcntl(fd, F_GETLK, &lock_data);

  if( len == 0 ) {
    for( i=0; i<6; i++ ) {
      printf("--------------------------\n");
      switch(i) {
      case 0:
len = 0x0;
break;
      case 1:
len = INT32_MAX;
break;
      case 2:
len = UINT32_MAX;
break;
      case 3:
len = INT64_MAX;
break;
      case 4:
len = (1L<<41);
break;
      case 5:
len = (1L<<41)+1;
break;
      default:
len = 0x0;
      }
      printf("Locking file %s with len 0x%lx\n",filename,len);
      status = lock_file(fd,len);
    }
    printf("--------------------------\n");
  }
  else {
    printf("Locking file %s with len 0x%lx\n",filename,len);
    status = lock_file(fd,len);
  }
  
  status = close(fd);
  return EXIT_SUCCESS;
}

static int lock_file(int fd, long len) {

  int status;

  lock_data.l_type = F_WRLCK;
  lock_data.l_whence = 0;
  lock_data.l_len = len;

  status = fcntl(fd, F_SETLK, &lock_data);

  printf("fcntl() returned %d\n",status);

  if( status == -1 ) {
    fprintf(stderr,"fcntl() returned error code %d: %s\n",
errno, strerror(errno));
  }
  
  lock_data.l_type = F_UNLCK;
  lock_data.l_whence = 0;
  lock_data.l_len = len;
  
  status = fcntl(fd, F_SETLK, &lock_data);

  return status;
}
---------------------------------------------------------------------

strace output:

fcntl64(17, F_SETLK64, {type=F_WRLCK, whence=SEEK_SET, start=0,
len=9223372036854775807}, 0xbfffd0e0)

where len argument is set to maximum of long type ( 263 - 1 ==
9223372036854775807 ).