Layering issue

What causes the layering to change with simply a change to filter?

HTML

<div id="container">
	<div><p>Text</p></div>
	<div></div>
</div>

<button id="toggle">Toggle</button>

CSS

#container {
	font-size: 0;
	font-family: sans-serif;
}
#container div {
	background: crimson;
	width: 100px;
	height: 50px;
	display: inline-block;
	position: relative;
	font-size: 24px;
}
p {
	position: absolute;
	top: 0;
	left: 0;
	margin: 0;
	line-height: 50px;
	width: 200%;
	text-align: center;
	z-index: 2;
}
button {
	margin-top: 10px;
	display: block;
}

JavaScript

const div1 = document.querySelectorAll("#container div")[0];
const div2 = document.querySelectorAll("#container div")[1];

let on = false;

document.querySelector("#toggle").addEventListener("click", function() {
	const value = on ? "brightness(1)" : "brightness(2)";
	div1.style.filter = div2.style.filter = value;

	on ^= true;
});