Reinforcement learning of targeted movement (Chadderdon et al. 2012)

 Download zip file   Auto-launch 
Help downloading and running models
Accession:144538
"Sensorimotor control has traditionally been considered from a control theory perspective, without relation to neurobiology. In contrast, here we utilized a spiking-neuron model of motor cortex and trained it to perform a simple movement task, which consisted of rotating a single-joint “forearm” to a target. Learning was based on a reinforcement mechanism analogous to that of the dopamine system. This provided a global reward or punishment signal in response to decreasing or increasing distance from hand to target, respectively. Output was partially driven by Poisson motor babbling, creating stochastic movements that could then be shaped by learning. The virtual forearm consisted of a single segment rotated around an elbow joint, controlled by flexor and extensor muscles. ..."
Reference:
1 . Chadderdon GL, Neymotin SA, Kerr CC, Lytton WW (2012) Reinforcement learning of targeted movement in a spiking neuronal model of motor cortex. PLoS One 7:e47251 [PubMed]
Citations  Citation Browser
Model Information (Click on a link to find other models with that property)
Model Type: Realistic Network;
Brain Region(s)/Organism: Neocortex;
Cell Type(s): Neocortex fast spiking (FS) interneuron; Neocortex spiking regular (RS) neuron; Neocortex spiking low threshold (LTS) neuron;
Channel(s):
Gap Junctions:
Receptor(s): GabaA; AMPA; NMDA;
Gene(s):
Transmitter(s): Dopamine; Gaba; Glutamate;
Simulation Environment: NEURON;
Model Concept(s): Simplified Models; Synaptic Plasticity; Long-term Synaptic Plasticity; Reinforcement Learning; Reward-modulated STDP;
Implementer(s): Neymotin, Sam [Samuel.Neymotin at nki.rfmh.org]; Chadderdon, George [gchadder3 at gmail.com];
Search NeuronDB for information about:  GabaA; AMPA; NMDA; Dopamine; Gaba; Glutamate;
/
arm1d
README
drspk.mod *
infot.mod *
intf6_.mod *
intfsw.mod *
misc.mod *
nstim.mod *
stats.mod *
updown.mod *
vecst.mod *
arm.hoc
basestdp.hoc
col.hoc *
colors.hoc *
declist.hoc *
decmat.hoc *
decnqs.hoc *
decvec.hoc *
default.hoc *
drline.hoc *
filtutils.hoc *
geom.hoc
grvec.hoc *
hinton.hoc *
infot.hoc *
init.hoc
intfsw.hoc *
labels.hoc *
local.hoc *
misc.h *
mosinit.hoc
network.hoc
nload.hoc
nqs.hoc *
nqsnet.hoc *
nrnoc.hoc *
params.hoc
run.hoc
samutils.hoc *
sense.hoc *
setup.hoc *
sim.hoc
simctrl.hoc *
stats.hoc *
stim.hoc
syncode.hoc *
units.hoc *
xgetargs.hoc *
                            
// $Id: xgetargs.hoc,v 1.20 2008/05/11 17:19:11 billl Exp $

print "Loading xgetargs.hoc..."

xgahfl=0
{load_file("decvec.hoc")}
strdef mesg
objref xgabxl,xgabxo

//* xgetargs()
// xgetargs(panel_name,command,arg1[,arg2,...],defaults) // existing or new params -- flag 2
//   may have no named params in this case since just set with call to command
//   eg xgetargs("New","redo","do this","do that","1,2")
// xgetargs(panel_name,command,"a1,a2,..") // existing params -- flag 1
// xgetargs(panel_name,command,strlist)
// xgetargs(1,...) // dismiss after setting
// xgetargs(list,...) // list of helper functions returned by xgetclrfunc()
// eg xgetargs("Random session","newrand","# of patts","patt size  ","overlap   ","5,33,7")
// optional 1st arg flag=1 means to remove panel after command is called
// Union contains: o[0]=VBox, o[1]=argv, [o[2]=param name list] o[3]=orig update
//                 o[4]=helper functions
//                 x[0]=#args, x[1]=flag, x[2]=dismiss x[3-5] reserved for future use
//                 s=panel/button name,t=quit call,u=varlable,v=scratch
obfunc xgetargs () { local i,j,args,flag,dismiss,na localobj o,argv,vb,l,st,xo
  if (!isassigned(xgabxl)) xgabxl=new List()
  st=new String2()
  na=numarg()  
  dismiss=0 i=1 
  xgabxl.append(o=new Union())
  if (argtype(i)==0) { dismiss=$1 i+=1 }
  if (argtype(i)==1) {xo=$oi i+=1} else xo=xgetclrfunc()
  o.os(4,"funcs",xo) // list of helper functions
  o.xs(2,"dismiss",dismiss) // dismiss=1 means dismiss at end
  o.os(1,"argv",argv=new Vector(na))
  argv.resize(0)
  o.os(0,"vb",vb=new VBox())
  sprint(o.t,"xgaqt(%s)",o)
  vb.dismiss_action(o.t)
  o.s=$si i+=1 o.t=$si i+=1
  vb.intercept(1)
  xpanel(o.s)
  xvarlabel(o.u)
  sprint(o.v,"xgetexec(%s)",o)
  xbutton(o.s,o.v)
  excu("xgetbuttn",o.o[4],o)
  if (i==na) { // this should be a list of existing params
    o.xs(1,"flag",flag=1) // flag -- variables have names
    o.os(2,"plist",l=new List()) // list of param names
    if (argtype(i)==2) {
      split_interp=1
      split($si,l) // list of strings
      split($si,argv) // list of values
    } else if (argtype(i)==1) {
      for ltr(xo,$oi) {
        l.append(xo)
        argv.append(str2num(xo.s))
      }
    } else { printf("xgetargs ERRA\n") xpanel() return o=nil}
    args=argv.size
    if (args==0) { // create them
      for ltr(xo,l) {
        sprint(st.s,"%s=1",xo.s) execute(st.s)
        args=l.count argv.resize(args) argv.fill(1)
      }
    } else if (args!=l.count) {printf("xgetargs ERRB %d %d\n",args,l.count) xpanel() return o=nil}
    o.os(3,"orig",argv.c)   // save original values
    o.xs(0,"nargs",args)
    excu("xgetlabl",o.o[4],o)
    for j=0,args-1 {
      sprint(o.v,"%s.x[%d]",argv,j)
      sprint(st.s,"xgetchg(%s,%d)",o,j)
      xvalue(l.o(j).t, o.v, 1, st.s, 1)
    }
  } else {
    j=i i=na
    if (strm($si,"^[a-z]")) { // a named variable; should all be name vars
      o.xs(1,"flag",flag=3) // flag -- variables have names and routine is called with args
      split_interp=1
      split($si,argv)
      o.os(2,"plist",l=new List()) // list of param names
      split($si,l) // list of strings
    } else {
      split_interp=0
      split($si,argv)
      o.xs(1,"flag",flag=2) // flag -- variables may have no names; are just args to a function
    }
    i=j // restore i
    o.xs(0,"nargs",args=argv.size)
    if (args!=na-i) { printf("xgetargs ERRC: mismatch %d %d\n",args,na-i) xpanel() return o=nil }
    o.os(3,"orig",argv.c)   // save original values
    for j=0,args-1 {
      sprint(o.v,"%s.x[%d]",argv,j)
      sprint(st.s,"xgetchg(%s,%d)",o,j)
      xvalue($si,o.v,1,st.s,1)
      if (flag==3) l.o(j).t=$si
      i+=1
    }
  }
  o.u=o.s // start with this label
  // xbutton("Help (2 clicks)","xgah()")
  xpanel()
  vb.intercept(0)
  vb.full_request(1)
  vb.map(o.s)
  xgabxo=o // global for current xgab object
  return o
}

//* xgah() should provide help -- doesn't
proc xgah () { 
  if (xgahfl==1) {
    continue_dialog("Press help button first, then elsewhere for button-specific help")
    xgahfl=0
  } else xgahfl=1
}

//* xgetclrfunc() -- set up the callbacks as empty functions
obfunc xgetclrfunc () { local flag localobj st,xo,o
  flag=0
  if (numarg()==1) if (isobj($o1,"List")) flag=1
  if (flag) { // just clear the functions
    for ltr(xo,$o1) xo.t="" // clear
    return $o1
  } else { // create
    st=new String() o=new List()
    for scase(st,"xgetchg2","xgetexec2","xgaqt2","xgetlabl","xgetbuttn") {
      o.append(new String2(st.s)) }
    if (numarg()==1) $o1=o
    return o
  }
}

//* xgetchg() done after a value is changed
proc xgetchg () { local i localobj o
  o=$o1 i=$2
print "A",o,i,o.o[4]
  if (excu("xgetchg2",o.o[4],o,i)) return
  sprint(o.u,"Press '%s' button for effect",o.s)
}

//* xgetexec() done when the 'make changes' button is pressed
proc xgetexec () { local i,args,dismiss,flag localobj o,l,av,vb,xo
  o=$o1
  if (excu("xgetexec2",o.o[4],o)) return
  args=o.x flag=o.x[1] dismiss=o.x[2] av=o.o[1] vb=o.o
  if (flag==1 || flag==3) {
    l=o.o[2]
    if (args!=l.count || args!=av.size) {
      printf("xgetexec() ERRA %d,%d,%d\n",args,l.count,av.size) return
    }
    for ltr(xo,l,&i) {
      sprint(o.v,"%s=%g",xo.s,av.x[i])
      execute(o.v)
    }
    if (flag==1) { // call the routine -- else will call below
      if (strm(o.t,"[(]")) o.v=o.t else sprint(o.v,"%s(%s)",o.t,o) // no args passed to function
    }
  }
  if (flag==2 || flag==3) {
    sprint(o.v,"%s(",o.t)
    for i=0,args-2 sprint(o.v,"%s%g,",o.v,av.x[i])
    sprint(o.v,"%s%g)",o.v,av.x[i]) 
  }
  execute(o.v)
  o.u="Changes submitted"
  if (dismiss) { xgaqt(o) // get rid of the box
  } else { o.o[3].copy(o.o[1]) } // new set of origs
}
  
//* xgaqt() called when the panel is dismissed
proc xgaqt () { local x localobj av,vb,o
  o=$o1 
  if (excu("xgaqt2",o.o[4],o)) return
  av=o.o[1] vb=o.o
  vb.unmap()
  if ((x=xgabxl.index(o))!=-1) xgabxl.remove(x)
  if (xgabxo==o) xgabxo=nil
}