SSCCE:

package sample;

import javafx.application.Application;
import javafx.beans.property.SimpleStringProperty;
import javafx.beans.property.StringProperty;
import javafx.collections.FXCollections;
import javafx.collections.ObservableList;
import javafx.geometry.Insets;
import javafx.scene.Scene;
import javafx.scene.control.Button;
import javafx.scene.control.ComboBox;
import javafx.scene.layout.HBox;
import javafx.stage.Stage;

public class Main extends Application {
@Override
public void start(final Stage primaryStage) throws Exception {
final ObservableList<Dummy> data =
FXCollections.observableArrayList(
new Dummy("Dummy 1"),
new Dummy("Dummy 2"),
new Dummy("Dummy 3"),
new Dummy("Dummy 4"),
new Dummy("Dummy 5"),
new Dummy("Dummy 6"),
new Dummy("Dummy 7"),
new Dummy("Dummy 8"),
new Dummy("Dummy 9")
);
final ComboBox<Dummy> comboBox = new ComboBox<>();
comboBox.setItems(data);
comboBox.setValue(data.get(4));
final Button deleteButton = new Button("Remove second");
deleteButton.setOnAction(ev -> data.remove(1));
final Button insertButton = new Button("Insert first");
insertButton.setOnAction(ev -> data.add(0, new Dummy("Dummy 0")));
final HBox root = new HBox(10D);
root.setPadding(new Insets(10D));
root.getChildren().addAll(comboBox, deleteButton, insertButton);
primaryStage.setTitle("ComboBox Test");
primaryStage.setScene(new Scene(root, 400D, 50D));
primaryStage.show();
}

public static void main(String[] args) {
Application.launch(args);
}

private static class Dummy {
private final StringProperty name = new SimpleStringProperty();

public Dummy(final String name) {
this.setName(name);
}

public StringProperty nameProperty() {return this.name;}
public String getName() {return this.name.get();}
public void setName(final String name) {this.name.set(name);}

@Override
public String toString() {return this.getName();}
}
}

1. Run the sample application. ComboBox value will be set to 5th item in the list.
2. Click button "Remove second". ComboBox value will change to 3rd item in the list.
3. Click button "Insert first". ComboBox value will revert to 5th item in the list.

This only happens if setItems() method is used. If ObservableList is passed via constructor, everything works as expected (ComboBox value is not changed when underlying list changes).