Loader in a Layout takes up space when inactive

import QtQuick 2.8
import QtQuick.Layouts 1.3
import QtQuick.Controls 2.1

ApplicationWindow {
    id: window
    objectName: "window"
    width: 800
    height: 800
    visible: true

    ColumnLayout {
        id: panelColumnLayout
        spacing: 2
        anchors.right: parent.right

        Rectangle {
            id: colourPanel
            color: "#444"
            implicitWidth: 300
            implicitHeight: 250

            Layout.fillHeight: true
        }

        Loader {
            objectName: "tilesetSwatchLoader"
            active: false
            sourceComponent: Rectangle {
                color: "#444"
                implicitWidth: 300
                implicitHeight: 250
            }

            Layout.fillHeight: true
        }
//        Rectangle {
//            color: "#444"
//            implicitWidth: 300
//            implicitHeight: 250
//            visible: false

//            Layout.fillHeight: true
//        }

        Loader {
            objectName: "layersLoader"
            active: true
            sourceComponent: Rectangle {
                color: "#444"
                implicitWidth: 300
                implicitHeight: 250

                Layout.fillHeight: true
            }
        }
    }
}