JSFiddle - React, Tailwind, and code Playground

by replicateur

JavaScript

class UCB1 {
  constructor(n) {
    this.counts = Array(n).fill(0);
    this.values = Array(n).fill(0);
  }

  getRandomInt(max) {
    return Math.floor(Math.random() * max);
  }

  selectRandom() {
    let n = this.counts.length;
    return this.getRandomInt(n);
  }

  getRandomBinary(probability) {
    return Math.random() < probability ? 1 : 0;
  }

  select() {
    let n = this.counts.length;
    let ucbValues = Array(n).fill(0);
    for (let i = 0; i < n; i++) {
      let c = this.counts[i];
      let v = this.values[i];
      ucbValues[i] = c === 0 ? Infinity : v + Math.sqrt(2 * Math.log(c) / c);
    }
    return ucbValues.indexOf(Math.max(...ucbValues));
  }

  update(chosenArm, reward) {
    this.counts[chosenArm] = this.counts[chosenArm] + 1;
    let n = this.counts[chosenArm];
    let value = this.values[chosenArm];
    let newValue = ((n - 1) / n) * value + (1 / n) * reward;
    this.values[chosenArm] = newValue;
  }

  getBestArm() {
    let bestArm = this.values.indexOf(Math.max(...this.values));
    return bestArm;
  }
}

const numberOfArms = 5;
const decimals = 3;
const bandit = new UCB1(numberOfArms);
const rewards = Array.from({
  length: numberOfArms
}, () => Math.round((Math.random() + Number.EPSILON) * (decimals * 10)) / (decimals * 10));

const limit = 1000000;

for (let i = 0; i < limit; i++) {
  // Select the arm to play
  let selectedArm = bandit.selectRandom();
  //console.log(`Selected arm: ${selectedArm}`);

  // Simulate the reward received
  let reward = bandit.getRandomBinary(rewards[selectedArm]);
  //console.log(`Received reward: ${reward}`);

  // Update the arm with the received reward
  bandit.update(selectedArm, reward);
}

console.log('------------------------- LIMIT = ' + limit);
console.log(bandit.counts);
console.log(bandit.values);
console.log(bandit.getBestArm());
console.log('END');