QSystemTrayIcon - unable to reset context menu

#include "mainwindow.h"
#include "./ui_mainwindow.h"

#include <QSystemTrayIcon>
#include <QMenu>
#include <QPushButton>

MainWindow::MainWindow(QWidget *parent)
    : QMainWindow(parent)
    , ui(new Ui::MainWindow)
{
    ui->setupUi(this);

    trayMenu = new QMenu(this);
    trayMenu->addAction(tr("Quit"), qApp, &QCoreApplication::quit);

    trayIcon = new QSystemTrayIcon(QPixmap(":/banana.png"), this);
    trayIcon->setContextMenu(trayMenu);

    trayIcon->show();
}

MainWindow::~MainWindow()
{
    delete ui;
}

void MainWindow::on_pushButton_clicked()
{
    if (trayIcon->contextMenu()) {
        trayIcon->setContextMenu(nullptr);
        ui->pushButton->setText("Add Tray Menu");
    } else {
        trayIcon->setContextMenu(trayMenu);
        ui->pushButton->setText("Remove Tray Menu");
    }
}