import java.awt.BorderLayout;
import java.awt.Component;
import java.awt.HeadlessException;
import java.awt.event.ActionEvent;
import javax.swing.JButton;
import javax.swing.JDialog;
import javax.swing.JFrame;
import javax.swing.JOptionPane;
import javax.swing.SwingUtilities;

public class ModalBehind extends JFrame {

    public static void main(String... args) {
        JFrame.setDefaultLookAndFeelDecorated(true);
        JDialog.setDefaultLookAndFeelDecorated(true);

        SwingUtilities.invokeLater(() -> new ModalBehind().setVisible(true));
    }

    public ModalBehind() throws HeadlessException {
        super("Main Frame");
        getContentPane().setLayout(new BorderLayout());

        JButton button = new JButton("Open Dialog");
        button.addActionListener(e -> showDialog());
        getContentPane().add(button, BorderLayout.PAGE_END);
        pack();
    }

    private void showDialog() {
        JButton button = new JButton("Open Nested Dialog");
        button.addActionListener(this::showNestedDialog);
        JOptionPane.showMessageDialog(this, button, "Top Level Dialog", JOptionPane.PLAIN_MESSAGE);
    }

    private void showNestedDialog(ActionEvent e) {
        Component parent = this;
        // Uncommenting this resolves the issue but is not necessary when LookAndFeelDecorated is false
        // parent = (Component) e.getSource();
        JOptionPane.showMessageDialog(parent,
                "When LookAndFeelDecorated is true and parent is the main frame,"
                + " clicking the top level dialog will make this go behind.",
                "Nested Modal Dialog", JOptionPane.PLAIN_MESSAGE);
    }
}