The trick is to draw your own link as a path with an arc in it. It took me a bit to work with the syntax of the arc parameters to make everything work, and the key seemed to be that the arc could not start and end on the same thing. Here is the corresponding code that draws edges with each update.
function tick() { link.attr("d", function(d) { var x1 = d.source.x, y1 = d.source.y, x2 = d.target.x, y2 = d.target.y, dx = x2 - x1, dy = y2 - y1, dr = Math.sqrt(dx * dx + dy * dy), // Defaults for normal edge. drx = dr, dry = dr, xRotation = 0, // degrees largeArc = 0, // 1 or 0 sweep = 1; // 1 or 0 // Self edge. if ( x1 === x2 && y1 === y2 ) { // Fiddle with this angle to get loop oriented. xRotation = -45; // Needs to be 1. largeArc = 1; // Change sweep to change orientation of loop. //sweep = 0; // Make drx and dry different to get an ellipse // instead of a circle. drx = 30; dry = 20; // For whatever reason the arc collapses to a point if the beginning // and ending points of the arc are the same, so kludge it. x2 = x2 + 1; y2 = y2 + 1; } return "M" + x1 + "," + y1 + "A" + drx + "," + dry + " " + xRotation + "," + largeArc + "," + sweep + " " + x2 + "," + y2; });
And here is a jsfiddle that demonstrates all this and a screenshot:

mes5k source share