Since this is the service that hosts the overlay window, this is a bit complicated solution, but it is possible.
You must handle these two cases separately (redefinition of pressing the "home" button and the "Back" button).
1. Pressing the "Home" button:
HomeWatcher, BroadcastReceiver, . , .
Android:
onCreate :
HomeWatcher mHomeWatcher = new HomeWatcher(this);
mHomeWatcher.setOnHomePressedListener(new OnHomePressedListener() {
    @Override
    public void onHomePressed() {
        yourWindow.hide()  
    }
    @Override
    public void onHomeLongPressed() {
    }
});
mHomeWatcher.startWatch();
2. "" :
,
( XML-).
, :
public class MyWindow
{
    private WindowManager windowManager;
    private WindowManager.LayoutParams params;
    private View view;
    
    private MyLayout myLayout;
    public MyWindow()
    {
        windowManager = (WindowManager) context.getSystemService(context.WINDOW_SERVICE);
        LayoutInflater inflater = (LayoutInflater) context.getSystemService(Context.LAYOUT_INFLATER_SERVICE);
        view = inflater.inflate(R.layout.your_original_window_layout, null);
        
        myLayout = new MyLayout(this);
        myLayout.addView(view, new ViewGroup.LayoutParams(ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.WRAP_CONTENT));
    }
    
    public void show()
    {
        windowManager.addView(myLayout, params);
    }
    public void hide()
    {
        windowManager.removeView(myLayout);
    }
}
MyLayout "" :
public class MyLayout extends LinearLayout
{
    private MyWindow myWindow;
    public MyLayout(MyWindow myWindow)
    {
        super(myWindow.context);
        this.myWindow = myWindow;
    }
    @Override public boolean dispatchKeyEvent(KeyEvent event)
    {
        if (event.getKeyCode() == KeyEvent.KEYCODE_BACK)
        {
            if (event.getAction() == KeyEvent.ACTION_DOWN  &&  event.getRepeatCount() == 0)
            {
                getKeyDispatcherState().startTracking(event, this);
                return true;
            }
            else if (event.getAction() == KeyEvent.ACTION_UP)
            {
                getKeyDispatcherState().handleUpEvent(event);
                if (event.isTracking() && !event.isCanceled())
                {
                    
                    myWindow.hide();
                    return true;
                }
            }
        }
        return super.dispatchKeyEvent(event);
    }
}
, , , , , . , .
.