Run the test case below on the Windows on the HiDPI screen when the TaskBar is enabled. Note that the toggled window is shifted too much from the Taskbar. The root cause of the bug is that we do not scale the screen insets from the pixels to the units.

import java.awt.BorderLayout;
import java.awt.Color;
import java.awt.Graphics;
import java.awt.GraphicsConfiguration;
import java.awt.GraphicsDevice;
import java.awt.GraphicsEnvironment;
import java.awt.Insets;
import java.awt.Rectangle;

import javax.swing.JButton;
import javax.swing.JComponent;
import javax.swing.JFrame;
import javax.swing.SwingUtilities;

public class UndecoratedJFrame extends JFrame {

    public static void main( String[] args ) {
        UndecoratedJFrame frame = new UndecoratedJFrame();
        frame.setVisible( true );
    }

    @Override
    public void setExtendedState( int state ) {
        if( state == MAXIMIZED_BOTH ) {
            // calculate setMaximizedBounds
            GraphicsConfiguration graphicsConfiguration = getGraphicsConfiguration();
            Rectangle screenBounds = graphicsConfiguration.getBounds();
            Insets in = getToolkit().getScreenInsets( graphicsConfiguration );

            Rectangle maxBounds = new Rectangle( screenBounds.x+in.left, screenBounds.y+in.top, screenBounds.width - in.left - in.right, screenBounds.height - in.top - in.bottom );
            setMaximizedBounds( maxBounds );
            super.setExtendedState( MAXIMIZED_BOTH );

            // since Java 9 we need calculate the scale factor because setMaximizedBounds reduce the resulting bounds with the display scale factor
            Rectangle bounds = getBounds();
            if( bounds.width != maxBounds.width || bounds.height != maxBounds.height ) {
                double factorX = (double)maxBounds.width / bounds.width;
                double factorY = (double)maxBounds.height / bounds.height;
                maxBounds = new Rectangle( in.left, in.top, (int)Math.round( screenBounds.width * factorX ) - in.left - in.right, (int)Math.round( screenBounds.height * factorY ) - in.top - in.bottom );
                setMaximizedBounds( maxBounds );
                super.setExtendedState( ICONIFIED ); // state must change that the new MaximizedBounds is used from the GUI
                SwingUtilities.invokeLater( () -> {
                    UndecoratedJFrame.super.setExtendedState( MAXIMIZED_BOTH );
                } );
            }
        } else {
            super.setExtendedState( state );
        }
    }

    /**
     * Create a test frame
     */
    UndecoratedJFrame() {
        setUndecorated( true );
        setSize(500,500);
        setLocationRelativeTo(null);
        setTitle( "setMaximizedBounds Test" );
        setDefaultCloseOperation( DISPOSE_ON_CLOSE );

        // find a position in the other monitor
        for( GraphicsDevice gd : GraphicsEnvironment.getLocalGraphicsEnvironment().getScreenDevices() ) {
            GraphicsConfiguration gc = gd.getDefaultConfiguration();
            Rectangle gcBounds = gc.getBounds();
            if( gcBounds.x != 0 || gcBounds.y != 0 ) {
                setBounds( gcBounds.x + 100, gcBounds.y + 100, 250, 150 );
                break;
            }
        }

        // we use 2 buttons instead a full L&F to make it simple
        // Toggle button
        JButton button = new JButton( "Toggle" );
        button.addActionListener( ( e ) -> {
            if( getExtendedState() == MAXIMIZED_BOTH ) {
                setExtendedState( NORMAL );
            } else {
                setExtendedState( MAXIMIZED_BOTH );
            }
        } );
        getContentPane().add( button, BorderLayout.WEST );
        // Close Button
        button = new JButton( "Close" );
        button.addActionListener( ( e ) -> {
            dispose();
        } );
        getContentPane().add( button, BorderLayout.EAST );
        getContentPane().add( new JComponent() {
            @Override
            public void paint( Graphics g ) {
                super.paint( g );
                // draw a red cross to see the size of the window
                g.setColor( Color.RED );
                g.drawLine( 0, 0, getWidth(), getHeight() );
                g.drawLine( 0, getHeight(), getWidth(), 0 );
            }
        }, BorderLayout.CENTER );
    }
}