Please compile/run the following code on HiDPI machine (or use sun.java2d.uiScale option):

import java.awt.*;
import java.awt.image.*;

public class TrayIconTest {

    private static BufferedImage generateImage(int w, int h, int scale, Color c) {

        int x = w * scale, y = h * scale;
        BufferedImage img = new BufferedImage(x, y, BufferedImage.TYPE_INT_RGB);
        Graphics g = img.getGraphics();
        g.setColor(c);
        g.fillRect(0, 0, x, y);
        g.setColor(Color.WHITE);
        g.fillRect(x / 3, y / 3, x / 3, y / 3);
        return img;
    }

    private static BaseMultiResolutionImage createIcon(int w, int h) {
        return new BaseMultiResolutionImage(
                new BufferedImage[]{generateImage(w, h, 1, Color.RED),
                                    generateImage(w, h, 2, Color.GREEN)});
    }
    
    public static void main(String[] args) throws Exception {
        
        if (!SystemTray.isSupported()) {
            System.err.println("tray is not supported");
            return;
        }
        
        SystemTray tray = SystemTray.getSystemTray();
        Dimension d = tray.getTrayIconSize();
        System.out.println("icon size: " + d.width + "x" + d.height);

        tray.add(new TrayIcon(createIcon(d.width, d.height)));
    }
}

expected: high-resolution ("green") icon will apear in the tray. in fact - the "red" one appears.

Win 8 + HiDPI, JDK9 b106