Reinforcement learning of targeted movement (Chadderdon et al. 2012)

 Download zip file   Auto-launch 
Help downloading and running models
Accession:144538
"Sensorimotor control has traditionally been considered from a control theory perspective, without relation to neurobiology. In contrast, here we utilized a spiking-neuron model of motor cortex and trained it to perform a simple movement task, which consisted of rotating a single-joint “forearm” to a target. Learning was based on a reinforcement mechanism analogous to that of the dopamine system. This provided a global reward or punishment signal in response to decreasing or increasing distance from hand to target, respectively. Output was partially driven by Poisson motor babbling, creating stochastic movements that could then be shaped by learning. The virtual forearm consisted of a single segment rotated around an elbow joint, controlled by flexor and extensor muscles. ..."
Reference:
1 . Chadderdon GL, Neymotin SA, Kerr CC, Lytton WW (2012) Reinforcement learning of targeted movement in a spiking neuronal model of motor cortex. PLoS One 7:e47251 [PubMed]
Citations  Citation Browser
Model Information (Click on a link to find other models with that property)
Model Type: Realistic Network;
Brain Region(s)/Organism: Neocortex;
Cell Type(s): Neocortex fast spiking (FS) interneuron; Neocortex spiking regular (RS) neuron; Neocortex spiking low threshold (LTS) neuron;
Channel(s):
Gap Junctions:
Receptor(s): GabaA; AMPA; NMDA;
Gene(s):
Transmitter(s): Dopamine; Gaba; Glutamate;
Simulation Environment: NEURON;
Model Concept(s): Simplified Models; Synaptic Plasticity; Long-term Synaptic Plasticity; Reinforcement Learning; Reward-modulated STDP;
Implementer(s): Neymotin, Sam [Samuel.Neymotin at nki.rfmh.org]; Chadderdon, George [gchadder3 at gmail.com];
Search NeuronDB for information about:  GabaA; AMPA; NMDA; Dopamine; Gaba; Glutamate;
/
arm1d
README
drspk.mod *
infot.mod *
intf6_.mod *
intfsw.mod *
misc.mod *
nstim.mod *
stats.mod *
updown.mod *
vecst.mod *
arm.hoc
basestdp.hoc
col.hoc *
colors.hoc *
declist.hoc *
decmat.hoc *
decnqs.hoc *
decvec.hoc *
default.hoc *
drline.hoc *
filtutils.hoc *
geom.hoc
grvec.hoc *
hinton.hoc *
infot.hoc *
init.hoc
intfsw.hoc *
labels.hoc *
local.hoc *
misc.h *
mosinit.hoc
network.hoc
nload.hoc
nqs.hoc *
nqsnet.hoc *
nrnoc.hoc *
params.hoc
run.hoc
samutils.hoc *
sense.hoc *
setup.hoc *
sim.hoc
simctrl.hoc *
stats.hoc *
stim.hoc
syncode.hoc *
units.hoc *
xgetargs.hoc *
                            
// $Id: drline.hoc,v 1.41 2011/02/15 14:05:02 billl Exp $

print "Loading drline.hoc..."

// click and drag left button to draw lines on top of a figure interactively
// select graph to draw on with setdrl(Graph[])
// set color with clr, line width with lne
// select 'Draw curve' for continuous drawing
// select 'Arrow' to place an arrow pointing according to direction of drag

drlflush=1 //whether to flush line drawings each drline call

//* drline(x0,y0,x1,y1,OPT graph or color) 
proc drline () { local color,line
  if (numarg()==0) { print "drline(x0,y0,x1,y1[,g,col,line])"
    return }
  if (numarg()>4) { 
    if (argtype(5)==0) { color=$5 
                         if (numarg()>5) line=$6
    } else {             graphItem = $o5 
                         if (numarg()>5) color=$6
                         if (numarg()>6) line=$7      }}
  graphItem.beginline(color,line)
  graphItem.line($1,$2)
  graphItem.line($3,$4)
  if(drlflush) graphItem.flush()
}

//* set to drawlines on top of fig
proc setdrl () {
  g=$o1 // select this graph for further drawing
  xpanel("")
  $o1.menu_tool("Draw line","drl")
  $o1.menu_tool("Draw curve","drc")
  $o1.menu_tool("Label","drw")
  $o1.menu_tool("Arrow","dra")
  $o1.menu_tool("Circle","drci")
  $o1.menu_tool("Rectangle","drr")
  xvalue("Color","clr",1,"",1)
  xvalue("Line","lne",1,"",1)
  xbutton("Erase","g.erase_all()")
  xpanel()
  $o1.exec_menu("Draw line")
}

//* draw line interactively on top of fig
// interesting that this should work at all since x0,y0 local but still preserving their
// values across multiple calls
proc drl ()  { local x0,y0,type,x,y,keystate
  type=$1 x=$2 y=$3 keystate=$4
  if (type==2) {x0=x y0=y}
  if (type==3) drline(x0,y0,x,y,clr,lne)
}

//* draw circle interactively on top of fig
// drci(2,0,0,0) drci(3,1,0,0)
proc drci ()  { local a,x0,y0,type,x,y,keystate,ii,rad localobj xv,yv
  type=$1 x=$2 y=$3 keystate=$4
  if (type==2) {x0=x y0=y}
  if (type==3) { rad=sqrt((x-x0)^2+(y-y0)^2) 
    a=allocvecs(xv,yv) vrsz(360,xv,yv)
    print "Circle: ",x0,y0,rad
    yv.circ(xv,x0,y0,rad)
    yv.line(g,xv,clr,lne)
    dealloc(a)
  }
}

//* draw retangle interactively on top of fig
proc drr ()  { local x0,y0,type,x,y,keystate
  type=$1 x=$2 y=$3 keystate=$4
  if (type==2) {x0=x y0=y}
  if (type==3) { drline(x0,y0,x0,y,clr,lne)
    drline(x,y0,x,y,clr,lne) drline(x,y,x0,y,clr,lne) drline(x,y0,x0,y0,clr,lne) }
}

//* draw arrow interactively on top of fig
proc dra ()  { local xsz,ysz,type,x,y,keystate,rot
  type=$1 x=$2 y=$3 keystate=$4
  xsz=0.1*(g.size(2)-g.size(1)) // 10% of size
  ysz=0.1*(g.size(4)-g.size(3))
  if (type==2) {x0=x y0=y}
  if (type==3) {
    if (y==y0) {
      if (x>x0) rot=-90 else rot=90
    } else {
      rot=-atan((x-x0)/(y-y0))/2/PI*360
      if ((y-y0)<=0) rot+=180
    }
    g.glyph(arrow(),x,y,xsz,ysz,rot)
  }
}

//* draw curve interactively on top of fig
proc drc ()  { local x0,y0,type,x,y,keystate
  type=$1 x=$2 y=$3 keystate=$4
  if (type==2) { x0=x y0=y
  } else if (type==1) {
    drline(x0,y0,x,y,clr,lne)
    x0=x y0=y
  } else if (type==3) drline(x0,y0,x,y,clr,lne)
}

//* write label
proc drw ()  { local x0,y0,type,x,y,keystate
  type=$1 x=$2 y=$3 keystate=$4
  if (type==2) { 
   string_dialog("Label: ",tstr) 
   g.label(x,y,tstr,1,1,0.5,0.5,clr)
  }
}

obfunc arrow () { localobj o
  o=new Glyph()
  o.m(0,0)  o.l(0,2) o.s(1,4) // draw vertical line
  o.m(0,0)  o.l(0,-2) o.s(1,4) // draw vertical line
  o.m(0,-2) o.l(-2,0) o.s(1,4)
  o.m(0,-2) o.l(2,0) o.s(1,4)
  return o
}

//* hist(g,vec,min,max,bins)
{clr=1 hflg=1 ers=1 sym=1 pflg=0 lin=4 hbup=0} 
declared("hfunc")
// clr:color, hflg=1 draw lines; 2 draw boxes; 3 fill in; ers=erase; 
// pflg=1 normalize hist by size of $o2, so will be probability instead of count
// pflg=2 turn hist upside down
// pflg=3 operate on values with hfunc()
// style determined by hflg
// hflg==0 lines with dots
// hflg==0.x offset lines with dots
// hflg==1 outlines but not down to zero
// hflg==2 outlines with lines down to zero
// hflg==3 just dots
// hflg==3.x lines between dots
// hbup=1 // move baseline up by this amount
func hist () { local a,b,c,min,max,wid,bins,ii,jj,offset,x,y
  if (numarg()==0) { printf("hist(g,vec,min,max,bins)\n") return 0}
  if ($o2.size<2)  { printf("hist: $o2 too small\n",$o2) return -1}
  if ($o2.min==$o2.max)  { printf("hist: %s all one value: %g\n",$o2,$o2.min) return -1}
  if (numarg()==5) {min=$3 max=$4 bins=$5 
  } else if (numarg()==4) { min=0 max=$3 bins=$4 
  } else if (numarg()<=3) { 
    if ((min=0.95*$o2.min)<0) min=1.05*$o2.min
    if ((max=1.05*$o2.max)<0) max=0.95*$o2.max
    bins=100
    if (min>0) min*=0.9 else min*=1.1
    if (max>0) max*=1.1 else max*=0.9
    if (numarg()==3) bins=$3
  }
  wid=(max-min)/bins
  // print min,max,max-wid,wid
  a=b=c=allocvecs(3) b+=1 c+=2
  offset=0 x=-1
  if (ers) $o1.erase_all()
  mso[c].hist($o2,min,bins,wid) // c has values
  if(pflg==1) mso[c].div(mso[c].sum) // normalize to sum to 1
  if(pflg==2) mso[c].mul(-1)
  if(pflg==3) hfunc(mso[c])
  mso[a].resize(2*mso[c].size())
  mso[a].indgen(0.5) 
  mso[a].apply("int") 
  mso[b].index(mso[c], mso[a]) 
  mso[a].mul(wid) mso[a].add(min)
  mso[b].rotate(1)
  mso[b].x[0] = 0 
  mso[b].append(mso[b].x[mso[b].size-1],0)
  mso[b].add(hbup)
  mso[a].append(max,max)
  if (hflg==1 || hflg==2) { 
    mso[b].line($o1, mso[a],clr,lin)
    if (hflg==2) for vtr(&x,mso[a]) drline(x,0,x,mso[b].x[i1],$o1,clr,lin)
  } else if (int(hflg)==0 || hflg>=3) { 
    if (hflg%1!=0) offset=hflg*wid // use eg -0.5+ii/8 to move back to integer
    mso[a].indgen(min,max-wid,wid)
    mso[a].add(wid/2+offset)
    // print mso[a].min,mso[a].max
    // mso[c].mark($o1,mso[a],"O",6,clr,2) // this will place points where 0 count
    for jj=0,mso[a].size-1 if (mso[c].x[jj]!=0) {
      if (hflg!=3 && hflg%1!=0) drline(mso[a].x[jj],0,mso[a].x[jj],mso[c].x[jj],$o1,clr,lin)
      if (hflg==4) {
        if (x!=-1) drline(x,y,mso[a].x[jj],mso[c].x[jj],$o1,clr,lin)
        x=mso[a].x[jj] y=mso[c].x[jj]
      }
      $o1.mark(mso[a].x[jj],mso[c].x[jj],sg(sym).t,10,clr,2) // don't place points with 0 count
    }
  }
  $o1.flush()
  $o1.size(min,max,0,mso[b].max)
  dealloc(a)
  return 1
}

// barplot(g,yvec,xvec[,bar_width]) 
// barplot(g,yvec,xvec[,bar_width,color_vec]) -- for multicolored bars -- each point has a color
// barplot(g,yvec,xvec[,bar_width,color_vec,error_vec]) -- error_vec plots the error
scribble=0
func barplot () { local a,sz,wid,ii,jj,x,y,mulcol localobj go,vx,vy,v1,vcol
  if (numarg()==0) {
    printf("barplot(g,yvec,xvec[,bar_width]), scribble=1 to 'fill in'\n") 
    printf("set scribble=1 to fill in with single color (based on clr)\n")
    printf("barplot(g,yvec,xvec[,bar_width,color_vec]):multicolored bars-each point has a color\n")
    printf("barplot(g,yvec,xvec[,bar_width,color_vec,error_vec]):add +/- error to each bar\n")
    return 0}
  if ((sz=$o2.size)!=$o3.size)  { printf("barplot: x,y vectors differ in size\n") return -1}
  go=$o1 $o3.sort
  if (argtype(4)==0)  wid=$4 else wid=1
  if (argtype(5)==1)  {vcol=$o5 mulcol=-1
    if (sz!=vcol.size) { printf("barplot: color vec wrong size: %d %d\n",sz,vcol.size) return -1}  
  } else if (argtype(5)==0) mulcol=$5 else mulcol=0
  wid/=2
  // print min,max,max-wid,wid
  a=allocvecs(vx,vy,v1)
  if (ers) go.erase_all()
  for vtr2(&x,&y,$o3,$o2,&ii)  { 
    vx.append(x-wid,x-wid,x+wid,x+wid)
    vy.append(0,y,y,0)
  }
  if (mulcol) {
    for vtr2(&x,&y,$o3,$o2,&jj)  { 
      if (mulcol==-1) clr=vcol.x[jj] else clr=mulcol
      vrsz(0,vx,vy)
      vx.append(x-wid,x-wid)
      vy.append(0,y)
      for (ii=0;ii<2*wid;ii+=(wid/100)) { 
        vx.add(wid/100) 
        vy.line(go, vx, clr, 4)
      }
    }
    vy.line(go, vx, clr, 4)
  } else if (scribble) {
    vrsz(0,vx,vy)
    for vtr2(&x,&y,$o3,$o2,&ii)  { 
      vx.append(x-wid,x-wid,x-wid)
      vy.append(0,y,0)
    }
    for (ii=0;ii<2*wid;ii+=(wid/100)) { 
      vx.add(wid/100) 
      vy.line(go, vx, clr, 4)
    }
    vy.line(go, vx, clr, 4)
  } else vy.line(go, vx, clr, lne)
  if(numarg()>5) $o2.ploterr(go, $o3, $o6, 15, 1, 3)
  go.flush()
  go.size(vx.min-wid,vx.max+wid,0,vy.max)
  dealloc(a)
  return 1
}

proc smgs () { local a,b,c,min,max,wid,bins,ii,jj,offset,x,y localobj v1
  if ($o2.size<2)  { printf("smgs: $o2 too small\n",$o2) return -1}
  if ($o2.min==$o2.max)  { printf("smgs: %s all one value: %g\n",$o2,$o2.min) return -1}
  if (numarg()==5) {min=$3 max=$4 bins=$5 
  } else if (numarg()==4) { min=0 max=$3 bins=$4 
  } else if (numarg()<=3) { 
    if ((min=0.95*$o2.min)<0) min=1.05*$o2.min
    if ((max=1.05*$o2.max)<0) max=0.95*$o2.max
    bins=100
    if (min>0) min*=0.9 else min*=1.1
    if (max>0) min*=1.1 else max*=0.9
    if (numarg()==3) bins=$3
  }
  wid=(max-min)/bins
  // print min,max,max-wid,wid
  a=b=c=allocvecs(3,1e4) b+=1 c+=2
  offset=0 x=-1
  if (ers) $o1.erase_all()
  mso[a].indgen(min,max,wid)
  if (0) {
    mso[c].smgs($o2,min,max,wid,wid*wid/4) // c has values
    mso[c].line($o1, mso[a],clr,4)
  } else {
    v1=$o2.sumgauss(min,max,wid,wid/2) // c has values
    v1.line($o1, mso[a],clr,4)
  }
}

//* a few drawing utilities from sam (not too spectacular)
 
//** drawhticks(ticksz,minx,maxx,linewidth,$5-$numarg() == y position of horizontal ticks)
// draw horizontal ticks of a view box along left/right of box
proc drawhticks () { local ticksz,minx,maxx,lw,i
  ticksz=$1 minx=$2 maxx=$3 lw=$4
  for i=5,numarg() {
    drline(minx,$i,minx+ticksz,$i,g,1,lw)    drline(maxx,$i,maxx-ticksz,$i,g,1,lw)
  }
}

//** drawvticks(ticksz,miny,maxy,linewidth,$5-$numarg() == x position of vertical ticks)
// draw vertical ticks of a view box along top/bottom of box
proc drawvticks () { local ticksz,miny,maxy,lw,i
  ticksz=$1 miny=$2 maxy=$3 lw=$4
  for i=5,numarg() {
    drline($i,miny,$i,miny+ticksz,g,1,lw)    drline($i,maxy,$i,maxy-ticksz,g,1,lw)
  }
}

//** drawbox(minx,maxx,miny,maxy[,line,graph]) - draw box
proc drawbox () { local minx,maxx,miny,maxy,ln localobj myg
  minx=$1 maxx=$2 miny=$3 maxy=$4
  if(numarg()>4)ln=$5 else ln=3
  if(numarg()>5)myg=$o6 else myg=g
  drline(minx,miny,minx,maxy,myg,1,ln) //bottom
  drline(minx,miny,maxx,miny,myg,1,ln) //left
  drline(minx,maxy,maxx,maxy,myg,1,ln) //top
  drline(maxx,miny,maxx,maxy,myg,1,ln) //right
}