Source Code for Module ML.Neural.Trainers

  1  ## Automatically adapted for numpy.oldnumeric Jun 27, 2008 by -c 
  2   
  3  # 
  4  #  Copyright (C) 2000-2008  greg Landrum 
  5  # 
  6  """ Training algorithms for feed-forward neural nets 
  7   
  8    Unless noted otherwise, algorithms and notation are taken from: 
  9    "Artificial Neural Networks: Theory and Applications", 
 10      Dan W. Patterson, Prentice Hall, 1996 
 11     
 12  """ 
 13  import numpy 
 14     
 15 -class Trainer(object): 
 16    """ "virtual base class" for network trainers 
 17   
 18    """ 
 19    pass 
 20   
 21 -class BackProp(Trainer): 
 22    """implement back propagation (algorithm on pp 153-154 of Patterson) 
 23   
 24     I don't *think* that I've made any assumptions about the connectivity of 
 25       the net (i.e. full connectivity between layers is not required). 
 26   
 27     **NOTE:** this code is currently making the assumption that the activation 
 28       functions on the nodes in the network are capable of calculating their 
 29       derivatives using only their values (i.e. a DerivFromVal method should 
 30       exist).  This shouldn't be too hard to change. 
 31      
 32    """ 
 33 -  def StepUpdate(self,example,net,resVect=None): 
 34      """ does a BackProp step based upon the example 
 35   
 36        **Arguments** 
 37   
 38          - example: a 2-tuple: 
 39             1) a list of variable values values 
 40             2) a list of result values (targets) 
 41   
 42          - net: a _Network_ (or something supporting the same API) 
 43   
 44          - resVect: if this is nonzero, then the network is not required to 
 45            classify the _example_ 
 46   
 47        **Returns** 
 48   
 49          the backprop error from _network_ **before the update** 
 50   
 51        **Note** 
 52   
 53          In case it wasn't blindingly obvious, the weights in _network_ are modified 
 54          in the course of taking a backprop step. 
 55           
 56      """ 
 57      totNumNodes = net.GetNumNodes() 
 58      if self.oldDeltaW is None: 
 59        self.oldDeltaW = numpy.zeros(totNumNodes,numpy.float64) 
 60      outputNodeList = net.GetOutputNodeList() 
 61      nOutput = len(outputNodeList) 
 62      targetVect = numpy.array(example[-nOutput:],numpy.float64) 
 63      trainVect = example[:-nOutput] 
 64      if resVect is None: 
 65        # classify the example 
 66        net.ClassifyExample(trainVect) 
 67        resVect = net.GetLastOutputs() 
 68      outputs = numpy.take(resVect,outputNodeList) 
 69      errVect = targetVect - outputs 
 70   
 71      delta = numpy.zeros(totNumNodes,numpy.float64) 
 72      # start with the output layer 
 73      for i in xrange(len(outputNodeList)): 
 74        idx = outputNodeList[i] 
 75        node = net.GetNode(idx) 
 76        # the deltas here are easy 
 77        delta[idx] = errVect[i]*node.actFunc.DerivFromVal(resVect[idx]) 
 78        # use these results to start working on the deltas of the preceding layer 
 79        inputs = node.GetInputs() 
 80        weights = delta[idx]*node.GetWeights() 
 81        for j in xrange(len(inputs)): 
 82          idx2 = inputs[j] 
 83          delta[idx2] = delta[idx2] + weights[j] 
 84   
 85      # now propagate the deltas backwards 
 86      for layer in xrange(net.GetNumHidden()-1,-1,-1): 
 87        nodesInLayer = net.GetHiddenLayerNodeList(layer) 
 88        for idx in nodesInLayer: 
 89          node = net.GetNode(idx) 
 90          # start by finishing off the error term for this guy 
 91          delta[idx] = delta[idx]*node.actFunc.DerivFromVal(resVect[idx]) 
 92   
 93          # and then propagate our errors to the preceding layer 
 94          if layer != 0: 
 95            inputs = node.GetInputs() 
 96            weights = delta[idx]*node.GetWeights() 
 97            for i in xrange(len(inputs)): 
 98              idx2 = inputs[i] 
 99              delta[idx2] = delta[idx2] + weights[i] 
100           
101      # okey dokey... we've now got the deltas for each node, use those 
102      #  to update the weights (whew!) 
103      nHidden = net.GetNumHidden() 
104      for layer in xrange(0,nHidden+1): 
105        if layer == nHidden: 
106          idxList = net.GetOutputNodeList() 
107        else: 
108          idxList = net.GetHiddenLayerNodeList(layer) 
109        for idx in idxList: 
110          node = net.GetNode(idx) 
111          dW = self.speed * delta[idx] * numpy.take(resVect,node.GetInputs()) 
112          newWeights = node.GetWeights() + dW 
113          node.SetWeights(newWeights) 
114         
115      # return the RMS error from the OLD network 
116      return numpy.sqrt(errVect*errVect)[0] 
117       
118   
119 -  def TrainOnLine(self,examples,net,maxIts=5000,errTol=0.1,useAvgErr=1, 
120                    silent=0): 
121      """ carries out online training of a neural net 
122   
123        The definition of online training is that the network is updated after 
124          each example is presented. 
125   
126        **Arguments** 
127   
128          - examples: a list of 2-tuple: 
129             1) a list of variable values values 
130             2) a list of result values (targets) 
131   
132          - net: a _Network_ (or something supporting the same API) 
133   
134          - maxIts: the maximum number of *training epochs* (see below for definition) to be 
135            run 
136   
137          - errTol: the tolerance for convergence 
138   
139          - useAvgErr: if this toggle is nonzero, then the error at each step will be 
140            divided by the number of training examples for the purposes of checking 
141            convergence. 
142   
143          - silent: controls the amount of visual noise produced as this runs. 
144   
145   
146        **Note** 
147   
148           a *training epoch* is one complete pass through all the training examples 
149   
150      """     
151      nExamples = len(examples) 
152      converged = 0 
153      cycle = 0 
154   
155      while (not converged) and (cycle < maxIts): 
156        maxErr = 0 
157        newErr = 0 
158        #print 'bp: ',cycle 
159        for example in examples: 
160          localErr = self.StepUpdate(example,net) 
161          newErr += localErr 
162          if localErr > maxErr: 
163            maxErr = localErr 
164        if useAvgErr == 1: 
165          newErr = newErr / nExamples 
166        else: 
167          newErr = maxErr 
168        #print '\t',newErr,errTol 
169           
170        if newErr <= errTol: 
171          converged = 1 
172   
173  #      if cycle % 10 == 0 and not silent: 
174        if not silent: 
175          print 'epoch %d, error: % 6.4f'%(cycle,newErr) 
176   
177        cycle = cycle + 1 
178      if not silent: 
179        if converged: 
180          print 'Converged after %d epochs.'%cycle 
181        else: 
182          print 'NOT Converged after %d epochs.'%cycle 
183        print 'final error: % 6.4f'%newErr 
184   
185 -  def __init__(self,speed=0.5,momentum=0.7): 
186      """ Constructor 
187   
188        **Arguments** 
189         
190          - speed: the speed parameter for back prop training 
191   
192          - momentum: the momentum term for back prop training 
193            *Not currently used* 
194   
195      """ 
196      self.speed = speed 
197      self.momentum = momentum 
198      self.oldDeltaW = None 
199   
200   
201   
202  if __name__ == '__main__': 
203    from ML.Neural import Network 
204   
205 -  def testAnd(): 
206      examples = [ 
207            [[0,0,1], [0.1]], 
208            [[0,1,1], [.1]], 
209            [[1,0,1], [.1]], 
210            [[1,1,1], [.9]] 
211        ] 
212      net = Network.Network([3,1]) 
213      t = BackProp() 
214      t.TrainOnLine(examples,net) 
215      return net 
216   
217 -  def testOr(): 
218      examples = [ 
219            [[0,0,1], [0.1]], 
220            [[0,1,1], [.9]], 
221            [[1,0,1], [.9]], 
222            [[1,1,1], [.9]] 
223        ] 
224      net = Network.Network([3,1]) 
225      t = BackProp() 
226      t.TrainOnLine(examples,net,maxIts=1000,useAvgErr=0) 
227      print 'classifications:' 
228      for example in examples: 
229        res = net.ClassifyExample(example[0]) 
230        print '%f -> %f'%(example[1][0],res) 
231   
232      return net 
233   
234 -  def testXor(): 
235      examples = [ 
236            [[0,0,1], [.1]], 
237            [[0,1,1], [.9]], 
238            [[1,0,1], [.9]], 
239            [[1,1,1], [.1]] 
240        ] 
241      net = Network.Network([3,3,1]) 
242   
243      t = BackProp(speed=.8) 
244      t.TrainOnLine(examples,net,errTol=0.2) 
245      return net 
246   
247   
248 -  def testLinear(): 
249      examples = [ 
250            [.1,.1], 
251            [.2,.2], 
252            [.3,.3], 
253            [.4,.4], 
254            [.8,.8], 
255        ] 
256      net = Network.Network([1,2,1]) 
257      t = BackProp(speed=.8) 
258      t.TrainOnLine(examples,net,errTol=0.1,useAvgErr=0) 
259      print 'classifications:' 
260      for example in examples: 
261        res = net.ClassifyExample(example[:-1]) 
262        print '%f -> %f'%(example[-1],res) 
263   
264      return net 
265   
266 -  def runProfile(command): 
267      import random 
268      random.seed(23) 
269      import profile,pstats 
270      datFile = '%s.prof.dat'%(command) 
271      profile.run('%s()'%command,datFile) 
272      stats = pstats.Stats(datFile) 
273      stats.strip_dirs() 
274      stats.sort_stats('time').print_stats() 
275       
276    if 0: 
277      net = testXor() 
278      print 'Xor:', net 
279      import cPickle 
280      outF = open('xornet.pkl','wb+') 
281      cPickle.dump(net,outF) 
282      outF.close() 
283    else: 
284      #runProfile('testLinear') 
285      net = testLinear() 
286      #net = testOr() 
287