fullPage.js with scroll inside sections & sticky element: beta

issue when scrolling back up :/

HTML

<script src="https://cdnjs.cloudflare.com/ajax/libs/jQuery-slimScroll/1.3.7/jquery.slimscroll.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/fullPage.js/2.7.8/jquery.fullPage.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/fullPage.js/2.7.8/vendors/jquery.easings.min.js"></script>
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/fullPage.js/2.7.8/jquery.fullPage.css">
<body>
  <div id="fullpage">
    <div id="scrolling-elem-wrap" class="scrolling-elem-wrap">
    </div>
    <div id="section-1" class="section">
      <div id="fixed-elem-wrap-1" class="fixed-elem-wrap">
        <div id="fixed-elem" class="fixed-elem">
          My fixed element
          <br> My fixed element
          <br> My fixed element
          <br> My fixed element
          <br> My fixed element
          <br>
          <br>
          <br>
          <br> My fixed element
          <br> My fixed element
          <br> My fixed element
          <br> My fixed element
          <br>
          <br>
          <br>
          <br> Another text
          <br> Another text
          <br> Another text
          <br> Another text
          <br>
          <br>
          <br>
          <br> Another text
          <br> Another text
          <br> Another text
          <br> Another text
        </div>
      </div>
      <div class="intro">
        <h1>Scrolling inside sections</h1>
        <p>Easy and useful! Just make use of the option `scrollOverflow` for it and add the slimScroll plugin!
          <br>
        </p>

      </div>
    </div>
    <div id="section-2" class="section">
      <div id="fixed-elem-wrap-2" class="fixed-elem-wrap">
      </div>
      <h2>
        Section 2
      </h2>
      <p>Eu nec ferri molestie consequat, vel at alia dolore putant. Labore philosophia ut vix. In vis nostrud interesset appellantur, vis et tation feugiat scripserit. Te nec noster suavitate persecuti. Diceret erroribus cu vix, alii omnes ei sit. Sea...

CSS

h1 {
  font-size: 3em;
}

h2 {
  font-size: 2.4em;
}

h1,
h2,
p {
  color: white;
  padding: 30px;
}
.section-1 {
  z-index: 600;
}

.fixed-elem-wrap {
  position: relative;
  height: 0;
}

.scrolling-elem-wrap {
  position: fixed;
  /* top: 150px; TBCalc */
  left: 0;
  right: 0;
  z-index: 9;
}

.fixed-elem {
  position: absolute;
  z-index: 500;
  top: 100px;
  right: 20px;
  padding: 20px;
  background-color: rgba(250, 250, 250, 0.5);
  transition: top 0.5s cubic-bezier(0, 1.4, 1, 1);
}

JavaScript

$(document).ready(function() {

  $('#fullpage').fullpage({
    sectionsColor: ['#4A6FB1', '#939FAA', '#323539', '#49AAFF'],
    scrollOverflow: true,
    onLeave: function(index, nextIndex, direction) {
    	console.log(index);
      moveFixedElemToSlide(nextIndex, direction);
    }
  });

  function moveFixedElemToSlide(movingToIndex, direction) {
    var incrementation = (direction === 'down') ? 1 : -1;
    var movingFromIndex = movingToIndex - incrementation;

    var elemMoved = $('#fixed-elem-wrap-' + movingFromIndex).contents();

    var moveFromSectionId = '#section-' + movingFromIndex;
    var moveToSectionId = '#section-' + movingToIndex;
    console.log(moveToSectionId + ': ' + $(moveToSectionId).position().top);

    var topDistanceStart = $(moveFromSectionId).position().top;
    var topDistanceEnd = $(moveToSectionId).position().top;
    $('#scrolling-elem-wrap').append(elemMoved);
    $('#scrolling-elem-wrap')
      .css({
        'top': -1000
      });
    $('#scrolling-elem-wrap').animate({
      'top': topDistanceEnd
    }, 600, 'easeOutBack',
    function () {
	    $('#fixed-elem-wrap-' + movingToIndex).append(elemMoved);
    });

  }

});