NestedTableColumnHeader has a factory method createTableColumnHeader to allow for hooking in custom headers. This method is used whenever a new header is created, except in its constructor. This omission effectively prevents custom headers.

To see the effect, the example below (longish, but mostly just boiler-plate to build up the stack of custom skins/headers) has a custom header with elliptic shape and a custom nested header with bluish background to see them better.

Run and look
- expected: all 4 headers elliptic
- actual: the top-level header of the nested column is rectangular

Reason: the rectangular header is created manually in the constructor of the nested. This must be replaced by creation via the factory method

Fix options:
a) add a factory method that explicitly creates not-nested headers
b) change the implementation of the factory method to create a not-nested header if the tableColumn is the same as the nested's (untested!!)

Code:

import com.sun.javafx.scene.control.skin.NestedTableColumnHeader;
import com.sun.javafx.scene.control.skin.TableColumnHeader;
import com.sun.javafx.scene.control.skin.TableHeaderRow;
import com.sun.javafx.scene.control.skin.TableViewSkin;
import com.sun.javafx.scene.control.skin.TableViewSkinBase;

import javafx.application.Application;
import javafx.scene.Parent;
import javafx.scene.Scene;
import javafx.scene.control.Skin;
import javafx.scene.control.TableColumn;
import javafx.scene.control.TableColumnBase;
import javafx.scene.control.TableView;
import javafx.scene.layout.Background;
import javafx.scene.layout.BackgroundFill;
import javafx.scene.layout.BorderPane;
import javafx.scene.paint.Color;
import javafx.scene.shape.Ellipse;
import javafx.stage.Stage;

/**
 * NestedTableColumnHeader must use factory method for creating
 * TableColumns always.
 *
 * Does not in its constructor.
 *
 * @author Jeanette Winzenburg, Berlin
 */
@SuppressWarnings({ "rawtypes", "unchecked" })
public class BugNestedColumnHeaderFactory extends Application {

    public static class MyColumnHeader extends TableColumnHeader {

        public MyColumnHeader(TableViewSkinBase skin, TableColumnBase tc) {
            super(skin, tc);
            setShape(new Ellipse(10, 10));
        }
        
    }
    public static class MyNestedColumnHeader extends NestedTableColumnHeader {
        
        @Override
        protected TableColumnHeader createTableColumnHeader(
                TableColumnBase col) {
            return col.getColumns().isEmpty() ?
                    new MyColumnHeader(getTableViewSkin(), col) :
                    new MyNestedColumnHeader(getTableViewSkin(), col);
        }

        public MyNestedColumnHeader(TableViewSkinBase skin,
                TableColumnBase tc) {
            super(skin, tc);
            // background just to see better
            setBackground(new Background(new BackgroundFill(Color.ALICEBLUE, null, null)));
        }
        
    }
    
    public static class MyTableHeader extends TableHeaderRow {

        
        @Override
        protected NestedTableColumnHeader createRootHeader() {
            return new MyNestedColumnHeader(getTableSkin(), null);
        }

        public MyTableHeader(TableViewSkinBase skin) {
            super(skin);
        }
        
    }
    
    public static class MyTableViewSkin extends TableViewSkin {

        @Override
        protected TableHeaderRow createTableHeaderRow() {
            return new MyTableHeader(this);
        }

        public MyTableViewSkin(TableView tableView) {
            super(tableView);
        }
        
    }
    private Parent getContent() {
        TableView table = new TableView() {

            @Override
            protected Skin createDefaultSkin() {
                return new MyTableViewSkin(this);
            }
            
        };
        TableColumn single = new TableColumn("Single");
        TableColumn nested = new TableColumn("Nested");
        TableColumn childOne = new TableColumn("Child One");
        TableColumn childTwo = new TableColumn("Child Two");
        nested.getColumns().addAll(childOne, childTwo);
        table.getColumns().addAll(single, nested);
        BorderPane pane = new BorderPane(table);
        return pane;
    }

    @Override
    public void start(Stage primaryStage) throws Exception {
        primaryStage.setScene(new Scene(getContent(), 600, 400));
// URL uri = getClass().getResource("headers.css");
// primaryStage.getScene().getStylesheets().add(uri.toExternalForm());
// primaryStage.setTitle(FXUtils.version());
        primaryStage.show();
    }

    public static void main(String[] args) {
        launch(args);
    }
}