深層強化学習（DRL）によるマルチエージェントシステムの概要と実装例

import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F

class Actor(nn.Module):
    def __init__(self, state_dim, action_dim, hidden_dim):
        super(Actor, self).__init__()
        self.fc1 = nn.Linear(state_dim, hidden_dim)
        self.fc2 = nn.Linear(hidden_dim, hidden_dim)
        self.fc3 = nn.Linear(hidden_dim, action_dim)
    
    def forward(self, state):
        x = F.relu(self.fc1(state))
        x = F.relu(self.fc2(x))
        x = torch.tanh(self.fc3(x))
        return x

class Critic(nn.Module):
    def __init__(self, state_dim, action_dim, hidden_dim):
        super(Critic, self).__init__()
        self.fc1 = nn.Linear(state_dim + action_dim, hidden_dim)
        self.fc2 = nn.Linear(hidden_dim, hidden_dim)
        self.fc3 = nn.Linear(hidden_dim, 1)
    
    def forward(self, state, action):
        x = torch.cat([state, action], 1)
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        return x

class MADDPG:
    def __init__(self, num_agents, state_dim, action_dim, hidden_dim, lr_actor=1e-3, lr_critic=1e-3, gamma=0.99, tau=0.01):
        self.num_agents = num_agents
        self.actors = [Actor(state_dim, action_dim, hidden_dim) for _ in range(num_agents)]
        self.target_actors = [Actor(state_dim, action_dim, hidden_dim) for _ in range(num_agents)]
        self.critics = [Critic(state_dim*num_agents, action_dim*num_agents, hidden_dim) for _ in range(num_agents)]
        self.target_critics = [Critic(state_dim*num_agents, action_dim*num_agents, hidden_dim) for _ in range(num_agents)]
        self.actor_optimizers = [optim.Adam(actor.parameters(), lr=lr_actor) for actor in self.actors]
        self.critic_optimizers = [optim.Adam(critic.parameters(), lr=lr_critic) for critic in self.critics]
        self.gamma = gamma
        self.tau = tau

        for i in range(num_agents):
            self.target_actors[i].load_state_dict(self.actors[i].state_dict())
            self.target_critics[i].load_state_dict(self.critics[i].state_dict())

    def select_action(self, states):
        actions = [actor(torch.tensor(states[i]).float()) for i, actor in enumerate(self.actors)]
        return torch.stack(actions)

    def update(self, states, actions, rewards, next_states, dones):
        for i in range(self.num_agents):
            state = torch.tensor(states[i]).float()
            action = torch.tensor(actions[i]).float()
            reward = torch.tensor(rewards[i]).float()
            next_state = torch.tensor(next_states[i]).float()
            done = torch.tensor(dones[i]).float()

            # Update critic
            target_actions = torch.stack([self.target_actors[j](next_state) for j in range(self.num_agents)])
            target_q = torch.squeeze(self.target_critics[i](torch.cat([next_state] + target_actions, 1)))
            target_value = reward + self.gamma * (1 - done) * target_q.detach()
            value = torch.squeeze(self.critics[i](torch.cat([state, action], 1)))
            critic_loss = F.mse_loss(value, target_value)
            self.critic_optimizers[i].zero_grad()
            critic_loss.backward()
            self.critic_optimizers[i].step()

            # Update actor
            policy_loss = -self.critics[i](torch.cat([state, self.actors[i](state)], 1)).mean()
            self.actor_optimizers[i].zero_grad()
            policy_loss.backward()
            self.actor_optimizers[i].step()

            # Update target networks
            for param, target_param in zip(self.actors[i].parameters(), self.target_actors[i].parameters()):
                target_param.data.copy_(self.tau * param.data + (1 - self.tau) * target_param.data)
            for param, target_param in zip(self.critics[i].parameters(), self.target_critics[i].parameters()):
                target_param.data.copy_(self.tau * param.data + (1 - self.tau) * target_param.data)