qlearningAgents.py
from game import *
from learningAgents import ReinforcementAgent
from featureExtractors import *
import random,util,math
class QLearningAgent(ReinforcementAgent):
"""
Q-Learning Agent
Functions you should fill in:
- getQValue
- getAction
- getValue
- getPolicy
- update
Instance variables you have access to
- self.epsilon (exploration prob)
- self.alpha (learning rate)
- self.gamma (discount rate)
Functions you should use
- self.getLegalActions(state)
which returns legal actions
for a state
"""
def __init__(self, **args):
"You can initialize Q-values here..."
ReinforcementAgent.__init__(self, **args)
"*** YOUR CODE HERE ***"
def getQValue(self, state, action):
"""
Returns Q(state,action)
Should return 0.0 if we never seen
a state or (state,action) tuple
"""
"*** YOUR CODE HERE ***"
util.raiseNotDefined()
def getValue(self, state):
"""
Returns max_action Q(state,action)
where is max is over legal actions
"""
"*** YOUR CODE HERE ***"
util.raiseNotDefined()
def getPolicy(self, state):
"""
What is the best action to take in a state
"""
"*** YOUR CODE HERE ***"
util.raiseNotDefined()
def getAction(self, state):
"""
What action to take in the current state. With
probability self.epsilon, we should take a random
action and take the best policy action otherwise.
After you choose an action make sure to
inform your parent self.doAction(state,action)
This is done for you, just don't clobber it
HINT: You might want to use util.flipCoin(prob)
HINT: To pick randomly from a list, use random.choice(list)
"""
# Pick Action
legalActions = self.getLegalActions(state)
action = None
"*** YOUR CODE HERE ***"
util.raiseNotDefined()
# Need to inform parent of action for Pacman (do not delete this line)
self.doAction(state,action)
return action
def update(self, state, action, nextState, reward):
"""
The parent class calls this to observe a
state = action => nextState and reward transition.
You should do your Q-Value update here
NOTE: You should never call this function,
it will be called on your behalf
"""
"*** YOUR CODE HERE ***"
util.raiseNotDefined()
class PacmanQAgent(QLearningAgent):
"Exactly the same as QLearningAgent, but with different default parameters"
def __init__(self, epsilon=0.05,gamma=0.8,alpha=0.2, numTraining=0, **args):
"""
These default parameters can be changed from the pacman.py command line.
For example, to change the exploration rate, try:
python pacman.py -p PacmanQLearningAgent -a epsilon=0.1
alpha - learning rate
epsilon - exploration rate
gamma - discount factor
numTraining - number of training episodes, i.e. no learning after these many episodes
"""
args['epsilon'] = epsilon
args['gamma'] = gamma
args['alpha'] = alpha
args['numTraining'] = numTraining
QLearningAgent.__init__(self, **args)
class ApproximateQAgent(PacmanQAgent):
"""
ApproximateQLearningAgent
You should only have to overwrite getQValue
and update. All other QLearningAgent functions
should work as is.
"""
def __init__(self, extractor='IdentityExtractor', **args):
self.featExtractor = util.lookup(extractor, globals())()
PacmanQAgent.__init__(self, **args)
# You might want to initialize weights here.
"*** YOUR CODE HERE ***"
def getQValue(self, state, action):
"""
Should return Q(state,action) = w * featureVector
where * is the dotProduct operator
"""
"*** YOUR CODE HERE ***"
util.raiseNotDefined()
def update(self, state, action, nextState, reward):
"""
Should update your weights based on transition
"""
"*** YOUR CODE HERE ***"
util.raiseNotDefined()
def final(self, state):
"Called at the end of each game."
# call the super-class final method
PacmanQAgent.final(self, state)
# did we finish training?
if self.episodesSoFar == self.numTraining:
# you might want to print your weights here for debugging
"*** YOUR CODE HERE ***"
pass
class BetterExtractor(FeatureExtractor):
"Your Mini-contest 2 entry goes here. Add features for capsuleClassic."
def getFeatures(self, state, action):
features = SimpleExtractor().getFeatures(state, action)
# Add more features here
"*** YOUR CODE HERE ***"
return features