There are two tricky areas for interop with SWT:
  - the event loop
  - child windows

The Event Loop

Fx does a great job at the lowest level making use of the event loop. Pulses and other low level Fx events are manifested as operating system messages. This allows Fx to use the event loop from another user inteface toolkit. Unfortunately, threading decisions are not exposed as API in Fx and are embedded in Fx classes (LauncherImpl, WinApplication.runLoop() etc.). Moving threading decisions out of the low level glass and all the way up to Application (even if not exposed in API) would allow for better interop.

Fx applications need to run the start() method on the UI thread and should be able to work no matter who starts the thread. Typically, SWT applications run their event loop in main() and the underlying Fx infrastructure could be changed to support this. SWT and Eclipse RCP applications cannot be easily changed and expect to run the loop.

Here is a proof of concept of SWT and Fx running in the same shell (NOTE: The code works on 32-bit cocoa too but Fx draws over the SWT button):

package fxinswt;

import java.lang.reflect.*;

import org.eclipse.swt.*;
import org.eclipse.swt.graphics.*;
import org.eclipse.swt.widgets.*;
import org.eclipse.swt.widgets.Button;

import com.sun.glass.ui.Window;
import com.sun.javafx.tk.quantum.*;

import javafx.application.*;
import javafx.beans.value.*;
import javafx.scene.*;
import javafx.scene.Group;
import javafx.scene.control.*;
import javafx.stage.*;

public class FxInSWT extends Application {
public void start(Stage stage) {
//Shared controls
final Button [] swtButton = new Button[1];
final CheckBox [] fxButton = new CheckBox[1];

//Create Fx controls
Group group = new Group();
Scene scene = new Scene(group, 350, 300);
/*CheckBox*/ fxButton[0] = new CheckBox("FX CheckBox");
fxButton[0].selectedProperty().addListener(new InvalidationListener() {
public void invalidated(ObservableValue ov) {
swtButton[0].setText("Hello SWT from Fx");
}
});
group.getChildren().add(fxButton[0]);
stage.setScene(scene);
stage.setVisible(true);

//Create SWT controls
long hwnd = getHandle (stage);
Display display = new Display();
final Shell shell = Shell.internal_new(display, (int)hwnd);
/*Button*/ swtButton[0] = new Button(shell, SWT.PUSH);
swtButton[0].setText("SWT Button");
swtButton[0].pack();
Rectangle rect1 = swtButton[0].getBounds();
swtButton[0].setBounds(30, 30, rect1.width + 60, rect1.height);
swtButton[0].addListener(SWT.Selection, new Listener() {
public void handleEvent (Event e) {
fxButton[0].setText("Hello Fx from SWT");
}
});
shell.setText("SWT set the title!");
shell.open();
while (!shell.isDisposed()) {
if (!display.readAndDispatch()) display.sleep();
}
display.dispose();
}

public static void main(String[] args) {
Launcher.launch(FxInSWT.class, null);
}

//TODO - this is utter crap
static long getHandle (Stage stage) {
WindowStage ws = ((WindowStage)stage.impl_getPeer());
Field platformWindow, ptr1;
try {
try {
platformWindow = WindowStage.class.getDeclaredField("platformWindow");
platformWindow.setAccessible(true);
Window ww = (Window) platformWindow.get(ws);
ptr1 = Window.class.getDeclaredField("ptr");
ptr1.setAccessible(true);
long hwnd = ptr1.getLong(ww);
return hwnd;
} catch (IllegalArgumentException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
} catch (IllegalAccessException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
} catch (SecurityException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
} catch (NoSuchFieldException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
return 0;
}
}


Child Windows

Fx renders to the client area of a top level window. This is a problem for heavyweight toolkits like SWT. While it is possible that applications could be coded to "leave a hole" for Fx to render into, this is not practical. Fortunately, it is easy to wrap an operating system handle, which is what happens now when AWT/Swing is embedded in SWT. If Fx were changed to optionally render to a child window, that window could be wrapped.

Here is a proof of concept of Fx rendering in a child windows (NOTE: Fx mouse and keys don't work due to the horribleness of the hack):

package colorfulcircles;

import java.lang.reflect.*;

import org.eclipse.swt.*;
import org.eclipse.swt.widgets.*;

import com.sun.glass.ui.*;
import com.sun.javafx.tk.quantum.*;

import javafx.animation.KeyFrame;
import javafx.animation.KeyValue;
import javafx.animation.Timeline;
import javafx.scene.Node;
import javafx.util.Duration;
import javafx.application.Application;
import javafx.application.Launcher;
import javafx.scene.Group;
import javafx.scene.Scene;
import javafx.scene.effect.BlendMode;
import javafx.scene.effect.BoxBlur;
import javafx.scene.paint.Color;
import javafx.scene.paint.CycleMethod;
import javafx.scene.paint.LinearGradient;
import javafx.scene.paint.Stop;
import javafx.scene.shape.Circle;
import javafx.scene.shape.Rectangle;
import javafx.scene.shape.StrokeType;
import javafx.stage.Stage;
import static java.lang.Math.random;
/**
 *
 * @author gchappel
 */
public class ColorfulCirclesSWT2 extends Application {

    @Override public void start(Stage stage) {

Group root = new Group();
Scene scene = new Scene(root, 800, 600, Color.BLACK);
        stage.setScene(scene);
        Group circles = new Group();
        for (int i = 0; i < 30; i++) {
            Circle circle = new Circle(150, Color.web("white", 0.05));
            circle.setStrokeType(StrokeType.OUTSIDE);
            circle.setStroke(Color.web("white", 0.16));
            circle.setStrokeWidth(4);
            circles.getChildren().add(circle);
        }
        Rectangle colors = new Rectangle(scene.getWidth(), scene.getHeight(),
                new LinearGradient(0f, 1f, 1f, 0f, true, CycleMethod.NO_CYCLE, new Stop[]{
                    new Stop(0, Color.web("#f8bd55")),
                    new Stop(0.14, Color.web("#c0fe56")),
                    new Stop(0.28, Color.web("#5dfbc1")),
                    new Stop(0.43, Color.web("#64c2f8")),
                    new Stop(0.57, Color.web("#be4af7")),
                    new Stop(0.71, Color.web("#ed5fc2")),
                    new Stop(0.85, Color.web("#ef504c")),
                    new Stop(1, Color.web("#f2660f")),}));
        Group blendModeGroup = new Group(
                new Group(new Rectangle(scene.getWidth(), scene.getHeight(),
                Color.BLACK), circles), colors);
        blendModeGroup.setBlendMode(BlendMode.OVERLAY);
        root.getChildren().add(blendModeGroup);
        circles.setEffect(new BoxBlur(10, 10, 3));
        Timeline timeline = new Timeline();
        for (Node circle : circles.getChildren()) {
            timeline.getKeyFrames().addAll(
                    new KeyFrame(Duration.ZERO, // set start position at 0s
                    KeyValue.keyValue(circle.translateXProperty(), random() * 800),
                    KeyValue.keyValue(circle.translateYProperty(), random() * 600)),
                    new KeyFrame(new Duration(40000), // set end position at 40s
                    KeyValue.keyValue(circle.translateXProperty(), random() * 800),
                    KeyValue.keyValue(circle.translateYProperty(), random() * 600)));
        }
        timeline.play();
        stage.setVisible(true);

//Create SWT controls
        long hwnd = getHandle(stage);
final Button [] swtButton = new Button[1];
Display display = new Display();
Shell shell = Shell.internal_new(display, (int)hwnd);
/*Button*/ swtButton[0] = new Button(shell, SWT.PUSH);
swtButton[0].setText("SWT Button");
swtButton[0].pack();
org.eclipse.swt.graphics.Rectangle rect1 = swtButton[0].getBounds();
swtButton[0].setBounds(30, 30, rect1.width + 60, rect1.height);
swtButton[0].addListener(SWT.Selection, new Listener() {
public void handleEvent (Event e) {
swtButton[0].setText("Hello Fx from SWT");
}
});
shell.setText("SWT set the title!");
//shell.pack();
shell.open();

Composite c = new Composite (shell, SWT.BORDER);
c.setBounds(30, 70, 400, 400);
hwndChild = c.handle;
getHandle(stage);

while (!shell.isDisposed()) {
if (!display.readAndDispatch()) display.sleep();
}
display.dispose();
    }

    /**
     * @param args the command line arguments
     */
    public static void main(String[] args) {
        Launcher.launch(ColorfulCirclesSWT2.class, args);
    }

//TODO - this is utter crap
static int hwndChild = 0;
static long getHandle (Stage stage) {
WindowStage ws = ((WindowStage)stage.impl_getPeer());
Field platformWindow, ptr1, ptr2;
try {
try {
platformWindow = WindowStage.class.getDeclaredField("platformWindow");
platformWindow.setAccessible(true);
Window ww = (Window) platformWindow.get(ws);
ptr1 = Window.class.getDeclaredField("ptr");
ptr1.setAccessible(true);
long hwnd = ptr1.getLong(ww);
View v = ww.getView();

//protected abstract void _setParent(long ptr, long parentPtr);
Class clazz = v.getClass();
ptr2 = View.class.getDeclaredField("ptr");
ptr2.setAccessible(true);
long viewHandle = ptr2.getLong(v);
try {
Method method = clazz.getDeclaredMethod("_setParent", long.class, long.class);
method.setAccessible(true);
if (hwndChild != 0) {
method.invoke(v, viewHandle, /*hwnd*/ hwndChild);
}
} catch (NoSuchMethodException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (InvocationTargetException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return hwnd;
} catch (IllegalArgumentException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
} catch (IllegalAccessException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
} catch (SecurityException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
} catch (NoSuchFieldException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
return 0;
}
}