polyfiddle
Polymer boilerplate for jsFiddle
HTML
<button>Open dialog inside Shadow DOM</button>
<div id="parent"></div>
JavaScript
let d = document.querySelector('#parent');
let shadowRoot = d.attachShadow({ mode: 'open' });
let child = document.createElement('dialog');
child.textContent = 'foo';
shadowRoot.appendChild(child);
let b = document.querySelector('button');
b.addEventListener('click', () => {
child.showModal();
});