FULL PRODUCT VERSION :
java version "1.8.0_152"
Java(TM) SE Runtime Environment (build 1.8.0_152-b16)
Java HotSpot(TM) 64-Bit Server VM (build 25.152-b16, mixed mode)

ADDITIONAL OS VERSION INFORMATION :
maxOS Sierra 10.12.6

A DESCRIPTION OF THE PROBLEM :
The mapping to screen coordinates is correct only for some rotations. The red square is mapped in 2d to the coordinate that localToScreen returns. It fails if the camera is rotated in Z and for some combinations if both X and Y are != 0.

Same problem in 9.0.1.

STEPS TO FOLLOW TO REPRODUCE THE PROBLEM :
Run the app and drag the sliders and it will be clear.

EXPECTED VERSUS ACTUAL BEHAVIOR :
EXPECTED -
The red square should always the connected to one corner (1, 1, 1).
ACTUAL -
The red square moves away from the corner.

REPRODUCIBILITY :
This bug can be reproduced always.

---------- BEGIN SOURCE ----------
import javafx.application.Application;
import javafx.geometry.Point2D;
import javafx.geometry.Point3D;
import javafx.scene.*;
import javafx.scene.control.Label;
import javafx.scene.control.Slider;
import javafx.scene.layout.HBox;
import javafx.scene.layout.Pane;
import javafx.scene.layout.VBox;
import javafx.scene.paint.Color;
import javafx.scene.shape.Box;
import javafx.scene.shape.Rectangle;
import javafx.scene.transform.Rotate;
import javafx.stage.Stage;

public class RotTest extends Application
{
private final PerspectiveCamera camera = new PerspectiveCamera(true);

private final Rotate rx = new Rotate(0, Rotate.X_AXIS);
private final Rotate ry = new Rotate(0, Rotate.Y_AXIS);
private final Rotate rz = new Rotate(0, Rotate.Z_AXIS);

private final Group grp3d = new Group();
private final Group camPiv = new Group();

private final Slider xSlider = new Slider(0, 360, 0);
private final Slider ySlider = new Slider(0, 360, 0);
private final Slider zSlider = new Slider(0, 360, 0);

private final Pane pane = new VBox();

private final Rectangle rect2d = new Rectangle(10, 10, Color.RED);

public void start(Stage stage) throws Exception
{
camPiv.getTransforms().addAll(rx, ry, rz); // X and y works, even in combo
// camPiv.getTransforms().addAll(rx, rz, ry); // X and y works, even in combo
// camPiv.getTransforms().addAll(rz, ry, rx); // X works and y works, but exclusively
// camPiv.getTransforms().addAll(rz, rx, ry); // X and y works, even in combo
// camPiv.getTransforms().addAll(ry, rz, rx); // X works and y works, but exclusively
// camPiv.getTransforms().addAll(ry, rx, rz); // X works and y works, but exclusively

grp3d.getChildren().add(camPiv);
camPiv.getChildren().add(camera);
camera.setTranslateZ(-10);

grp3d.getChildren().add(new Box(2, 2, 2)); // -1 to 1

SubScene subScene = new SubScene(grp3d, -1, -1, true, SceneAntialiasing.BALANCED);
subScene.setCamera(camera);

subScene.heightProperty().bind(pane.heightProperty());
subScene.widthProperty().bind(pane.widthProperty());

xSlider.valueProperty().addListener((o, oldA, newA) -> setAngle(rx, newA.doubleValue()));
ySlider.valueProperty().addListener((o, oldA, newA) -> setAngle(ry, newA.doubleValue()));
zSlider.valueProperty().addListener((o, oldA, newA) -> setAngle(rz, newA.doubleValue()));

pane.getChildren().addAll(
new HBox(new Label("X: "), xSlider),
new HBox(new Label("Y: "), ySlider),
new HBox(new Label("Z: "), zSlider),
subScene, rect2d
);

rect2d.setManaged(false);

stage.setScene(new Scene(pane));
stage.setWidth(1000);
stage.setHeight(1000);
stage.show();

setRectCoord();
}

private void setAngle(Rotate r, double a)
{
r.setAngle(a);
setRectCoord();
}

private void setRectCoord()
{
Point2D p = pane.screenToLocal(grp3d.localToScreen(new Point3D(1, 1, 1)));
rect2d.setLayoutX(p.getX());
rect2d.setLayoutY(p.getY());
}

public static void main(String[] args)
{
launch(args);
}
}

---------- END SOURCE ----------