JSFiddle - React, Tailwind, and code Playground

HTML

<select id="id_user" name="user" multiple="multiple">     <option value="1">User 1</option>     <option value="2">User 2</option>     <option value="3">User 3</option>     <option value="4">User 4</option> </select> 

<span class="item_select" data-item="1">Select 1 user</span> 
<span class="item_remove" data-item="1">Remove 1 select</span> 
<span class="item_select" data-item="2">Select 2 user</span> 
<span class="item_remove" data-item="2">Remove 2 select</span> 
<span class="item_select" data-item="3">Select 3 user</span>
<span  class="item_remove" data-item="3">Remove 3 select</span> 
<span class="item_select" data-item="4">Select 4 user</span> 
<span  class="item_remove" data-item="4">Remove 4 select</span>

CSS

.item_select,.item_remove{ display:block;margin:10px;}

JavaScript

var ctrlKey = false;

    $(document).on('keydown keyup',function(e) {
        if (e.which === 17) {
            ctrlKey = e.type=='keydown';
        }
    });
var $user = $('#id_user')
$('.item_select,.item_remove ').click(function() {
    var isSelect = $(this).is('.item_select')
    if (!ctrlKey && isSelect) {
        $user.val('')
    }
    $user.find('option[value="' + $(this).data('item') + '"]').prop('selected', isSelect)
});