JSFiddle - React, Tailwind, and code Playground

HTML

<div class="outer" id="scrollContainer">
  <div class="container">
    <div id="draggable"></div>
  </div>
</div>

CSS

.container {
  width: 5000px;
  height: 5000px;
  background: green;
}

.outer {
  width: 200px;
  height: 200px;
  overflow: scroll;
  margin-top: 100px;
  margin-left: 100px;
}

#draggable {
  width: 20px;
  height: 20px;
  background: blue;
  position: absolute;
}

JavaScript

const element = document.body.querySelector('#draggable');

const isInContainer = (x,y) => {
	const elements = document.elementsFromPoint(x, y)
  return elements.find(el => el && el.classList && el.classList.contains('container')) || false;
}
window.a =  document.getElementById('scrollContainer');

const onMouseMove = (e) => {
	if(isInContainer(e.pageX, e.pageY)){
    element.style.top = e.pageY + 'px';
    element.style.left = e.pageX + 'px';
  }
  
  const scrollBody = document.getElementById('scrollContainer');
  const rects = scrollBody.getBoundingClientRect();
  // check directions

  const maxX = scrollBody.scrollWidth - scrollBody.clientWidth;
  const maxY = scrollBody.scrollHeight - scrollBody.clientHeight;
  const canScrollTop = Math.round(scrollBody.scrollTop) > 0;
  const canScrollBottom = Math.round(scrollBody.scrollTop) < maxY;
  const canScrollLeft = Math.round(scrollBody.scrollLeft) > 0;
  const canScrollRight = Math.round(scrollBody.scrollLeft) < maxX;

  const x = e.pageX;
  const y = e.pageY;

  const minifier = 2;
  nextScrollDistance.y = 0;
  nextScrollDistance.x = 0;

  // scroll to bottom
  if (canScrollBottom && y > rects.bottom) {
    const distance = Math.abs(y - rects.bottom);
    nextScrollDistance.y = Math.round(distance / minifier)
    scrollPositions.down = true;
  } else {
    scrollPositions.down = false;
  }

  if (canScrollTop && y < rects.top) {
    const distance = Math.abs(y - rects.top);
    nextScrollDistance.y = Math.round(distance / minifier) * -1;
    scrollPositions.up = true;
  } else {
    scrollPositions.up = false;
  }

  if (canScrollRight && x > rects.right) {
    const distance = Math.abs(x - rects.right);
    nextScrollDistance.x = Math.round(distance / minifier)
    scrollPositions.right = true;
  } else {
    scrollPositions.right = false;
  }

  if (canScrollLeft && x < rects.left) {
    const distance = Math.abs(x - rects.left);
    nextScrollDistance.x = Math.round(distance / minifier) * -1;
   ...