There are at least two ways to achieve this ...
Could you...
Use JFrame#setShapeto change the shape of the main window, for example ...

JFrame frame = new JFrame("Testing");
frame.getContentPane().setBackground(Color.RED);
frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
frame.setLayout(new GridBagLayout());
frame.add(new JLabel("Boo!"));
frame.setSize(200, 200);
frame.setLocationRelativeTo(null);
frame.setUndecorated(true);
frame.setShape(new Ellipse2D.Double(0, 0, 200, 200));
frame.setVisible(true);
, , , . , ...
...
"" ...

import java.awt.Color;
import java.awt.Dimension;
import java.awt.EventQueue;
import java.awt.Graphics;
import java.awt.Graphics2D;
import java.awt.GridBagLayout;
import java.awt.RenderingHints;
import java.awt.geom.Ellipse2D;
import javax.swing.JFrame;
import javax.swing.JLabel;
import javax.swing.JPanel;
import javax.swing.UIManager;
import javax.swing.UnsupportedLookAndFeelException;
public class CircleUI {
public static void main(String[] args) {
new CircleUI();
}
public CircleUI() {
EventQueue.invokeLater(new Runnable() {
@Override
public void run() {
try {
UIManager.setLookAndFeel(UIManager.getSystemLookAndFeelClassName());
} catch (ClassNotFoundException | InstantiationException | IllegalAccessException | UnsupportedLookAndFeelException ex) {
}
JFrame frame = new JFrame("Testing");
frame.setUndecorated(true);
frame.setBackground(new Color(0, 0, 0, 0));
frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
frame.add(new CirclePane());
frame.pack();
frame.setLocationRelativeTo(null);
frame.setVisible(true);
}
});
}
public class CirclePane extends JPanel {
public CirclePane() {
setOpaque(false);
setLayout(new GridBagLayout());
add(new JLabel("Boo!"));
setBackground(Color.RED);
}
@Override
public Dimension getPreferredSize() {
return new Dimension(200, 200);
}
@Override
protected void paintComponent(Graphics g) {
super.paintComponent(g);
Graphics2D g2d = (Graphics2D) g.create();
g2d.setRenderingHint(RenderingHints.KEY_ALPHA_INTERPOLATION, RenderingHints.VALUE_ALPHA_INTERPOLATION_QUALITY);
g2d.setRenderingHint(RenderingHints.KEY_ANTIALIASING, RenderingHints.VALUE_ANTIALIAS_ON);
g2d.setRenderingHint(RenderingHints.KEY_COLOR_RENDERING, RenderingHints.VALUE_COLOR_RENDER_QUALITY);
g2d.setRenderingHint(RenderingHints.KEY_DITHERING, RenderingHints.VALUE_DITHER_ENABLE);
g2d.setRenderingHint(RenderingHints.KEY_FRACTIONALMETRICS, RenderingHints.VALUE_FRACTIONALMETRICS_ON);
g2d.setRenderingHint(RenderingHints.KEY_INTERPOLATION, RenderingHints.VALUE_INTERPOLATION_BILINEAR);
g2d.setRenderingHint(RenderingHints.KEY_RENDERING, RenderingHints.VALUE_RENDER_QUALITY);
g2d.setRenderingHint(RenderingHints.KEY_STROKE_CONTROL, RenderingHints.VALUE_STROKE_PURE);
g2d.setColor(Color.RED);
g2d.fill(new Ellipse2D.Double(0, 0, getWidth() - 1, getHeight() - 1));
g2d.dispose();
}
}
}
, , , ( ), , ..
...
...

JFrame frame = new JFrame("Testing");
frame.setUndecorated(true);
frame.setBackground(new Color(0, 0, 0, 0));
frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
JPanel content = new JPanel(new BorderLayout());
content.setOpaque(false);
content.setBorder(new EmptyBorder(1, 1, 1, 1));
content.add(new CirclePane());
frame.setContentPane(content);
frame.pack();
int width = frame.getWidth();
int height = frame.getHeight();
frame.setShape(new Ellipse2D.Double(0, 0, width, height));
frame.setLocationRelativeTo(null);
frame.setVisible(true);
, , setShape . JPanel, "" (1 ) . , , , , ...