Uploaded image for project: 'JDK'
  1. JDK
  2. JDK-8303614

(fs) WatchKey is not valid after registering 510 UNC paths

XMLWordPrintable

    • generic
    • windows_2012

      ADDITIONAL SYSTEM INFORMATION :
      Reproduced also on Windows Server 2016.
      Reproduced also with JDK8, JDK11, JDK17 and JDK19

      A DESCRIPTION OF THE PROBLEM :
      Create, run WatchService and wait for event (take) to monitor folders activities (create/delete/replace file) :
      1- Everything is working well without any limitation when registering folder using windows local path ex: E:\PROJET\WORKSPACE-DEVELOP\watwher-folder-test\TEST\MAIN-FOLDER\FolderTest_511.
      2- When sharing the parent folder : E:\PROJET\WORKSPACE-DEVELOP\watwher-folder-test\TEST, UNC path is : \\Dvosdbn011\test\MAIN-FOLDER\FolderTest_511. When using UNC path to register folder, take() method is not suspend anymore and WatchKey is not valid. No event can be detected on new regsitered UNC folder path

      STEPS TO FOLLOW TO REPRODUCE THE PROBLEM :
      Create more than 512 folders under a common folder
      Share common folder
      Start WatchService and run take() method in a separated thread
      Regsiter one by one folders

      EXPECTED VERSUS ACTUAL BEHAVIOR :
      EXPECTED -
      All folders are registered, all events generated by creating new files in any of the folders are received
      ACTUAL -
      When registering the 511th UNC folder path the take() method is not suspended anymore and the WatchKey is not valid, WatchKey buffer is null. Also all file created in folder that was regsitered after the first 510 folders is not detected.

      ------------------------------------------------------------
      Register folder : FolderTest_510
      Scanning \\Dvosdbn011\test\MAIN-FOLDER\FolderTest_510 ...
      Registered watcher : sun.nio.fs.WindowsWatchService@1e965684, key : sun.nio.fs.WindowsWatchService$WindowsWatchKey@4d95d2a2
      register: \\Dvosdbn011\test\MAIN-FOLDER\FolderTest_510
      Done.
      Number of watched folders : 510
      Number of invalid watched folders : 0
      ------------------------------------------------------------
      Register folder : FolderTest_511
      Scanning \\Dvosdbn011\test\MAIN-FOLDER\FolderTest_511 ...
      Registered watcher : sun.nio.fs.WindowsWatchService@1e965684, key : sun.nio.fs.WindowsWatchService$WindowsWatchKey@53f65459
      register: \\Dvosdbn011\test\MAIN-FOLDER\FolderTest_511
      Done.
      Number of watched folders : 511
      Number of invalid watched folders : 0
      ------------------------------------------------------------
      Register folder : FolderTest_512
      Scanning \\Dvosdbn011\test\MAIN-FOLDER\FolderTest_512 ...
      WATCHER TAKE ... Got it !
      number of events received : 0
      WATCHER TAKE ... keys.size = 510
      WATCHER Watch key is not valid : \\Dvosdbn011\test\MAIN-FOLDER\FolderTest_511
      WATCHER Watch key has been removed : \\Dvosdbn011\test\MAIN-FOLDER\FolderTest_511
      Registered watcher : sun.nio.fs.WindowsWatchService@1e965684, key : sun.nio.fs.WindowsWatchService$WindowsWatchKey@3b088d51
      register: \\Dvosdbn011\test\MAIN-FOLDER\FolderTest_512
      Done.
      Number of watched folders : 512
      Number of invalid watched folders : 1
      ------------------------------------------------------------
      Register folder : FolderTest_513
      Scanning \\Dvosdbn011\test\MAIN-FOLDER\FolderTest_513 ...
      Registered watcher : sun.nio.fs.WindowsWatchService@1e965684, key : sun.nio.fs.WindowsWatchService$WindowsWatchKey@1786dec2
      register: \\Dvosdbn011\test\MAIN-FOLDER\FolderTest_513
      Done.
      WATCHER TAKE ... Got it !
      WATCHER Watch key is not valid : \\Dvosdbn011\test\MAIN-FOLDER\FolderTest_512
      WATCHER Watch key has been removed : \\Dvosdbn011\test\MAIN-FOLDER\FolderTest_512
      number of events received : 0
      WATCHER TAKE ... keys.size = 511
      Number of watched folders : 513
      Number of invalid watched folders : 2
      ------------------------------------------------------------

      ---------- BEGIN SOURCE ----------
      package com.watcher.folder.test;

      import static com.sun.nio.file.SensitivityWatchEventModifier.HIGH;
      import static java.nio.file.LinkOption.NOFOLLOW_LINKS;
      import static java.nio.file.StandardWatchEventKinds.ENTRY_CREATE;
      import static java.nio.file.StandardWatchEventKinds.ENTRY_DELETE;
      import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY;
      import static java.nio.file.StandardWatchEventKinds.OVERFLOW;

      import java.io.File;
      import java.io.IOException;
      import java.nio.file.FileSystems;
      import java.nio.file.Files;
      import java.nio.file.Path;
      import java.nio.file.WatchEvent;
      import java.nio.file.WatchKey;
      import java.nio.file.WatchService;
      import java.util.HashMap;
      import java.util.Iterator;
      import java.util.Map;
      import java.util.Set;
      import java.util.concurrent.ExecutorService;
      import java.util.concurrent.Executors;
      import java.util.stream.Collectors;

      import com.google.common.collect.HashBiMap;

      public class FolderWatcherTest
      {

        /** The number of folders to watch **/
        private static final int MAX_NUMBER_FOLDERS = 515;

        /** The folder suffix number to stop **/
        private static final int STOP_NUMBER = 510;

        /** local folder path **/
        private static final String PARENT_FOLDER_1 = System.getProperty("user.dir") + File.separator + "TEST";

        /** shared folder path UNC **/
        private static final String PARENT_FOLDER_2 = "\\\\Dvosdbn011\\test";

        // ---------- //

        /** parent folder **/
        private File parentFolder = new File(PARENT_FOLDER_1);

        /** main folder **/
        private String mainFolder = "MAIN-FOLDER";

        /** The watcher. */
        private WatchService watcher;

        /** The is watcher closed. */
        private boolean isWatcherClosed = false;

        /** The phase name. */
        private Map<Path, String> dirNames;

        /** The keys. */
        private HashBiMap<WatchKey, Path> keys;

        /** The pool. */
        private ExecutorService pool;

        // ------------------------------------------ //

        public FolderWatcherTest()
        {
          dirNames = new HashMap<>();
          keys = HashBiMap.create();
          pool = Executors.newCachedThreadPool();

          runTest();
        }

        /**
         *
         */
        private void runTest()
        {
          System.out.println("----- START TEST -----");
          System.out.println("Working Directory = " + System.getProperty("user.dir"));

          try
          {
            createWatchFolder();

            File baseFolder = FolderWatcherTest.createFolder(parentFolder, mainFolder);
            this.setupTest(baseFolder);

            this.registerAll(baseFolder);

            this.cleanTest(baseFolder.getAbsolutePath());
            System.out.println("----- END TEST -----");
          }
          catch (Exception e)
          {
            e.printStackTrace();
          }
        }

        /**
         * @throws IOException
         */
        private void createWatchFolder() throws IOException
        {

          watcher = FileSystems.getDefault().newWatchService();

          pool.execute(new Runnable()
          {
            @Override
            public void run()
            {
              try
              {
                System.out.println("Process all events for keys queued to the watcher");
                Path dir = processEvents();
                displayNumberOfWatchedFolders();

                System.err.println("Interruption of watching directories ! " + dir);
              }
              catch (Exception e)
              {
                System.err.println(e.getMessage());
              }
            }
          });

        }

        /**
         * Display number of watched folders
         */
        private void displayNumberOfWatchedFolders()
        {
          int nbFolders = dirNames.size();
          System.out.println("Number of watched folders : " + nbFolders);
          System.out.println("Number of invalid watched folders : " + (nbFolders - keys.size()));
        }

        /**
         * Process all events for keys queued to the watcher.
         *
         * @throws InterruptedException
         * the interrupted exception
         */
        public Path processEvents() throws Exception
        {
          Path dir = null;
          while (!isWatcherClosed)
          {

            // wait for key to be signaled
            System.out.println("WATCHER TAKE ... keys.size = " + keys.size());
            WatchKey key = null;
            try
            {
              key = watcher.take();
            }
            catch (InterruptedException x)
            {
              System.err.println("WATCHER TAKE ERROR : InterruptedException");
              continue;
            }
            catch (Exception x2)
            {
              System.err.println("WATCHER TAKE ERROR : " + x2.getMessage());
              throw new Exception(x2);
            }

            System.out.println("WATCHER TAKE ... Got it !");

            dir = keys.get(key);
            if (dir == null)
            {
              System.err.println("WatchKey not recognized!!");
              continue;
            }

            String dirName = dirNames.get(dir);

            int nb_event_received = 0;
            for (WatchEvent< ? > event : key.pollEvents())
            {
              nb_event_received++;

              // get event type
              WatchEvent.Kind< ? > kind = event.kind();

              // Context for directory entry event is the file name of entry
              WatchEvent<Path> ev = cast(event);
              Path name = ev.context();
              Path child = dir.resolve(name);

              // print out event
              System.out.println("Kind of event [" + event.kind().name() + "] : name = " + name + ", child = " + child);

              // --------------------------------------------------- //
              if (kind == OVERFLOW)
              {
                System.out.println("[OVERFLOW] processing event on phase : " + dirName + " in folder : " + child.getParent()
                    + File.separator + child.getFileName());
                continue;
              }

              // --------------------------------------------------- //

              if (kind == ENTRY_CREATE)
              {
                System.out.println("[ENTRY_CREATE] processing event on phase : " + dirName + " in folder : "
                    + child.getParent() + File.separator + child.getFileName());
                if (Files.isRegularFile(child, NOFOLLOW_LINKS))
                {
                  System.out.println("ENTRY_CREATE");
                }
              }

              // --------------------------------------------------- //

              else if (kind == ENTRY_DELETE)
              {
                System.out.println("[ENTRY_DELETE] processing event on phase : " + dirName + " in folder : "
                    + child.getParent() + File.separator + child.getFileName());
              }

              // --------------------------------------------------- //

              // this event is catch after pasting new file in a folder or
              // after replacing existing file in a folder
              else if (kind == ENTRY_MODIFY)
              {
                System.out.println("[ENTRY_MODIFY] processing event on phase : " + dirName + " in folder : "
                    + child.getParent() + File.separator + child.getFileName());
              }
            }
            System.out.println("number of events received : " + nb_event_received);

            // reset key and remove from set if directory no longer accessible
            key.reset();

            boolean valid = key.isValid();
            if (!valid)
            {
              System.err.println("WATCHER Watch key is not valid : " + dir);
              keys.remove(key);
              System.err.println("WATCHER Watch key has been removed : " + dir);
              // all directories are inaccessible
              if (keys.isEmpty())
              {
                System.err.println("WATCHER No more watch key for this phase : " + dir);
                break;
              }
            }
          }

          System.err.println("WATCHER has been closed : " + dir);
          return dir;
        }

        /**
         * Cast.
         *
         * @param <T>
         * the generic type
         * @param event
         * the event
         * @return the watch event
         */
        @SuppressWarnings("unchecked")
        static <T> WatchEvent<T> cast(WatchEvent< ? > event)
        {
          return (WatchEvent<T>) event;
        }

        // ------------------------------------------ //

        /**
         * @param baseFolder
         * @throws Exception
         */
        private void registerAll(File baseFolder) throws Exception
        {
          File[] listFolder = baseFolder.listFiles();
          for (File folder : listFolder)
          {
            if (folder.isDirectory())
            {
              String folderSuffix = folder.getName().split("FolderTest_")[1];
              int folderIdx = Integer.parseInt(folderSuffix);
              boolean displayLog = folderIdx >= STOP_NUMBER;

              if (displayLog)
              {
                System.out.println("------------------------------------------------------------");
                System.out.println("Register folder : " + folder.getName());
              }

              register(folder.getName(), folder.toPath(), displayLog);

              if (displayLog)
              {
                displayNumberOfWatchedFolders();
              }

              if (("FolderTest_" + STOP_NUMBER).equals(folder.getName()))
              {
                System.out.println("---Break---");
              }
            }
          }
        }

        /**
         * Register a new path
         *
         * @param dirName
         * @param dir
         * @param displayLog
         * @throws IOException
         */
        private void register(String dirName, Path dir, boolean displayLog) throws IOException
        {
          if (displayLog)
          {
            System.out.println("Scanning " + dir + " ...");
          }

          dirNames.put(dir, dirName);

          WatchKey key = dir.register(watcher, new WatchEvent.Kind[]{ENTRY_CREATE, ENTRY_DELETE, ENTRY_MODIFY }, HIGH);

          if (displayLog)
          {
            System.out.println("Registered watcher : " + watcher.toString() + ", key : " + key);
          }

          Path prev = keys.get(key);
          if (prev == null && displayLog)
          {
            System.out.println("register: " + dir);
          }
          else
          {
            if (!dir.equals(prev) && displayLog)
            {
              System.out.println("update: -> " + dir);
            }
          }
          keys.put(key, dir);

          if (displayLog)
          {
            System.out.println("Done.");
          }
        }

        // ------------------------------------------ //

        /**
         * Setup test
         *
         * @param baseFolder
         * @throws Exception
         */
        private void setupTest(File baseFolder) throws Exception
        {
          String subFolder = "FolderTest_00";
          int i = 1;
          while (i < 10)
          {
            FolderWatcherTest.createFolder(baseFolder, subFolder + i);
            i++;
          }

          subFolder = "FolderTest_0";
          while (i < 100)
          {
            FolderWatcherTest.createFolder(baseFolder, subFolder + i);
            i++;
          }

          subFolder = "FolderTest_";
          while (i <= MAX_NUMBER_FOLDERS)
          {
            FolderWatcherTest.createFolder(baseFolder, subFolder + i);
            i++;
          }
        }

        /**
         * Clean the test once it has been finished by removing all folders
         *
         * @param folderToRemove
         * @throws Exception
         */
        private void cleanTest(String folderToRemove) throws Exception
        {
          unregisterAll(new File(folderToRemove));
          isWatcherClosed = true;
          FolderWatcherTest.deleteFolder(folderToRemove);
        }

        /**
         * Setup test
         *
         * @param baseFolder
         * @throws Exception
         */
        private void unregisterAll(File baseFolder) throws Exception
        {
          File[] listFolder = baseFolder.listFiles();
          for (File folder : listFolder)
          {
            if (folder.isDirectory())
            {
              unregister(folder.getName());
            }
          }
        }

        /**
         * Unregister all paths
         *
         * @param dirName
         * the phase name
         */
        private void unregister(String dirName)
        {
          Set<Path> paths = dirNames.entrySet().stream().filter(entry -> dirName.equals(entry.getValue()))
              .map(Map.Entry::getKey).collect(Collectors.toSet());

          Iterator<Path> it = paths.iterator();
          while (it.hasNext())
          {
            Path path = it.next();
            WatchKey key = keys.inverse().get(path);
            if (key != null)
            {
              key.cancel();
            }

            dirNames.remove(path);
            keys.remove(key);
          }
        }

        // ------------------------------------------ //

        /**
         * Creates the folder.
         *
         * @param parentFolder
         * the parent folder
         * @param folderNameToCreate
         * the folder name to create
         * @return the file
         * @throws UtilException
         * the util exception
         */
        public static File createFolder(File parentFolder, String folderNameToCreate) throws Exception
        {

          File folder = new File(parentFolder.getAbsoluteFile(), folderNameToCreate);

          if (!folder.exists())
          {
            if (folder.mkdir())
            {
              // System.out.println("Succeed to create folder : " + folder);
              return folder;
            }
            else
            {
              throw new Exception("Impossible to create folder " + folder.getAbsolutePath());
            }
          }
          else
          {
            return folder;
          }
        }

        /**
         * Delete folder and its content.
         *
         * @param folder
         * the folder
         * @return true, if successful
         */
        static public boolean deleteFolder(String folderPath)
        {
          boolean isRemoved = true;
          File folder = new File(folderPath);

          if (folder.exists())
          {
            if (folder.isDirectory())
            {
              deleteFolderOrFileContent(folder);
              deleteEmptyFolder(folder);
            }
            else
            {
              isRemoved = false;
              System.err.println("Folder is not a directory, can't remove folder : " + folder.getAbsolutePath());
            }
          }
          else
          {
            isRemoved = false;
            System.err.println("Folder does not exists, can't remove folder : " + folder.getAbsolutePath());
          }
          return isRemoved;
        }

        /**
         * Delete folder content.
         *
         * @param folder
         * the folder
         * @return true, if successful
         */
        static private boolean deleteFolderOrFileContent(File file)
        {
          boolean isRemoved = true;
          // delete file
          if (file.isFile())
          {
            try
            {
              Files.deleteIfExists(file.toPath());
              isRemoved = true;
            }
            catch (Exception e)
            {
              isRemoved = false;
              System.err.println("Impossible to remove " + file.getAbsolutePath() + " file : " + e.getMessage());
            }
          }

          else if (file.isDirectory())
          {
            File[] content = file.listFiles();
            if (content != null && content.length > 0) // some JVMs return null for empty dirs
            {
              for (File f : content)
              {
                if (f.isDirectory())
                {
                  isRemoved = deleteFolderOrFileContent(f);
                  deleteEmptyFolder(f);
                }
                else
                {
                  try
                  {
                    Files.deleteIfExists(f.toPath());
                    isRemoved = true;
                  }
                  catch (Exception e)
                  {
                    isRemoved = false;
                    System.err.println("Impossible to remove " + f.getAbsolutePath() + " : " + e.getMessage());
                  }
                }
              }
            }
            else
            {
              isRemoved = false;
              // System.out.println("Folder is empty : " + file.getAbsolutePath());
            }
          }

          else
          {
            isRemoved = false;
            System.err.println("Unknown type of content : " + file.getAbsolutePath());
          }

          return isRemoved;
        }

        static private boolean deleteEmptyFolder(File folder)
        {
          boolean isRemoved = true;

          // delete folder if it is empty
          if (folder.isDirectory() && (folder.list() == null || folder.list().length == 0))
          {
            boolean result = folder.delete();
            if (!result)
            {
              isRemoved = false;
              System.err.println("Impossible to remove " + folder.getAbsolutePath() + " folder");

              // try several times
              int nb_tries = 3;
              while (nb_tries-- > 0)
              {
                System.out.println("Folder still exists : " + folder.exists());
                // add timer of one second to let system to clear resources
                try
                {
                  Thread.sleep(1000);
                }
                catch (InterruptedException e)
                {
                  System.err.println(e.getMessage());
                }

                // try again
                System.out.println("Try again...");
                result = folder.delete();
                if (!result)
                {
                  isRemoved = false;
                  System.err.println("Impossible to remove " + folder.getAbsolutePath() + " folder");
                }
                else
                {
                  isRemoved = true;
                  break;
                }
              }
            }
          }
          else
          {
            isRemoved = false;
            System.err.println("Folder is not, can't remove folder : " + folder.getAbsolutePath());
          }
          return isRemoved;
        }

        // -------------------------------------------------- //

        /**
         * @param args
         */
        public static void main(String[] args)
        {
          new FolderWatcherTest();
          System.exit(0);
        }

      }

      ---------- END SOURCE ----------

      FREQUENCY : always


            bpb Brian Burkhalter
            webbuggrp Webbug Group
            Votes:
            0 Vote for this issue
            Watchers:
            4 Start watching this issue

              Created:
              Updated: