Reinforcement learning of targeted movement (Chadderdon et al. 2012)

 Download zip file   Auto-launch 
Help downloading and running models
Accession:144538
"Sensorimotor control has traditionally been considered from a control theory perspective, without relation to neurobiology. In contrast, here we utilized a spiking-neuron model of motor cortex and trained it to perform a simple movement task, which consisted of rotating a single-joint “forearm” to a target. Learning was based on a reinforcement mechanism analogous to that of the dopamine system. This provided a global reward or punishment signal in response to decreasing or increasing distance from hand to target, respectively. Output was partially driven by Poisson motor babbling, creating stochastic movements that could then be shaped by learning. The virtual forearm consisted of a single segment rotated around an elbow joint, controlled by flexor and extensor muscles. ..."
Reference:
1 . Chadderdon GL, Neymotin SA, Kerr CC, Lytton WW (2012) Reinforcement learning of targeted movement in a spiking neuronal model of motor cortex. PLoS One 7:e47251 [PubMed]
Citations  Citation Browser
Model Information (Click on a link to find other models with that property)
Model Type: Realistic Network;
Brain Region(s)/Organism: Neocortex;
Cell Type(s): Neocortex fast spiking (FS) interneuron; Neocortex spiking regular (RS) neuron; Neocortex spiking low threshold (LTS) neuron;
Channel(s):
Gap Junctions:
Receptor(s): GabaA; AMPA; NMDA;
Gene(s):
Transmitter(s): Dopamine; Gaba; Glutamate;
Simulation Environment: NEURON;
Model Concept(s): Simplified Models; Synaptic Plasticity; Long-term Synaptic Plasticity; Reinforcement Learning; Reward-modulated STDP;
Implementer(s): Neymotin, Sam [Samuel.Neymotin at nki.rfmh.org]; Chadderdon, George [gchadder3 at gmail.com];
Search NeuronDB for information about:  GabaA; AMPA; NMDA; Dopamine; Gaba; Glutamate;
/
arm1d
README
drspk.mod *
infot.mod *
intf6_.mod *
intfsw.mod *
misc.mod *
nstim.mod *
stats.mod *
updown.mod *
vecst.mod *
arm.hoc
basestdp.hoc
col.hoc *
colors.hoc *
declist.hoc *
decmat.hoc *
decnqs.hoc *
decvec.hoc *
default.hoc *
drline.hoc *
filtutils.hoc *
geom.hoc
grvec.hoc *
hinton.hoc *
infot.hoc *
init.hoc
intfsw.hoc *
labels.hoc *
local.hoc *
misc.h *
mosinit.hoc
network.hoc
nload.hoc
nqs.hoc *
nqsnet.hoc *
nrnoc.hoc *
params.hoc
run.hoc
samutils.hoc *
sense.hoc *
setup.hoc *
sim.hoc
simctrl.hoc *
stats.hoc *
stim.hoc
syncode.hoc *
units.hoc *
xgetargs.hoc *
                            
: $Id: misc.mod,v 1.24 2011/10/14 15:00:26 samn Exp $

COMMENT
Misc. routines:
sassign() // assign a string from system
dassign()// assign a double
nokill() // chatch SIGHUP
prtime() // gives date/time
fspitchar(c,file) // sends single char to a file
spitchar(c)       // sends single char to stdout: eg c=1 => ^A
file_exist(file) // returns 1 if filename exists
hocgetc(file) // get single char from a file

  Note that with a SUFFIX equal to "nothing" these functions do not
have a suffix in hoc.  Thus to call sassign() in hoc use simply type
"sassign()" <- without the quotes.

    file_exist(filename)
        - returns 1 if filename exists

    sassign()  (string assign, written by Bill Lytton)
        - This routine is used to set a string in Hoc to something that has
          been returned by a system call.  sassign("name","shell_call ...")
          will produce a file called "sassign" in the cwd that will contain
          a hoc call that sets string 'name' to the result of shell_call 
          which should be a string.
        
    dassign()  (double assign, written and used by Bill Lytton)
        - This routine is used to set a variable in Hoc to something that has
          been returned by a system call.  sassign("name","shell_call ...")
          will produce a file called "dassign" in the cwd that will contain
          a hoc call that sets variable 'name' to the result of shell_call 
          which should be a number.

ENDCOMMENT
                           
INDEPENDENT {t FROM 0 TO 1 WITH 1 (ms)}

NEURON {
    SUFFIX nothing
}

VERBATIM
#include <unistd.h>     /* F_OK     */
#include <errno.h>      /* errno    */
#include <signal.h>
#include <sys/types.h>         /* MUST REMEMBER THIS */
#include <time.h>
#include <stdio.h>
#include <limits.h>
#include "misc.h"
ENDVERBATIM

:* FUNCTION file_exist()
FUNCTION file_exist() {
VERBATIM
    /* Returns TRUE if file exists, if file not exist the need to reset
       errno else will get a nrnoc error.  Seems to be a problem even
       if I don't include <errno.h> */

    char *filename;

    filename = gargstr(1);

    if (*filename && !access(filename, F_OK)) {
        _lfile_exist = 1;

    } else {
        /* Errno set to 2 when file not found */
        errno = 0;

        _lfile_exist = 0;
    }
ENDVERBATIM
}

FUNCTION istmpobj () {
VERBATIM
  _listmpobj=hoc_is_tempobj_arg(1);
ENDVERBATIM  
}

:* PROCEDURE sassign()
FUNCTION sassign() {
VERBATIM
    FILE *pipein;
    char string[BUFSIZ], **strname, *syscall;

    strname = hoc_pgargstr(1);
    syscall = gargstr(2);

    if( !(pipein = popen(syscall, "r"))) {
        fprintf(stderr,"System call failed\n");
        return 0; 
    }
    
    if (fgets(string,BUFSIZ,pipein) == NULL) {
        fprintf(stderr,"System call did not return a string\n");
        pclose(pipein); return 0;
    }

    /*  assign_hoc_str(strname, string, 0); */
    hoc_assign_str(strname, string);

    pclose(pipein);
    errno = 0;
    return 0;
ENDVERBATIM
}

:* PROCEDURE dassign() 
FUNCTION dassign() {
VERBATIM
    FILE *pipein, *outfile;
    char *strname, *syscall;
    double num;

    strname = gargstr(1);
    syscall = gargstr(2);

    if ( !(outfile = fopen("dassign","w"))) {
        fprintf(stderr,"Can't open output file dassign\n");
        return 0; 
    }

    if( !(pipein = popen(syscall, "r"))) {
        fprintf(stderr,"System call failed\n");
        fclose(outfile); return 0; 
    }
    
    if (fscanf(pipein,"%lf",&num) != 1) {
        fprintf(stderr,"System call did not return a number\n");
        fclose(outfile); pclose(pipein); return 0; 
    }

    fprintf(outfile,"%s=%g\n",strname,num);
    fprintf(outfile,"system(\"rm dassign\")\n");

    fclose(outfile); pclose(pipein);
    errno = 0;
    return 0;
ENDVERBATIM
}

:* PROCEDURE nokill() 
: nohup
PROCEDURE nokill() {
VERBATIM
  signal(SIGHUP, SIG_IGN);
ENDVERBATIM
}

:* FUNCTION prtime()
FUNCTION prtime () {
VERBATIM
_lprtime = clock();
ENDVERBATIM
}

:* FUNCTION now ()
FUNCTION now () {
VERBATIM
  _lnow = time((time_t*)0);
  _lnow -= (12784) * 24*60*60; // time from the Epoch to 01/01/05
ENDVERBATIM
}

:* FUNCTION sleepfor (seconds[,nanoseconds])
: returns 0 on success, -1 on failure, nanosecond arg should be < 1 second
FUNCTION sleepfor (sec) {
VERBATIM
  struct timespec ts;
  ts.tv_sec = (time_t)_lsec;
  ts.tv_nsec = ifarg(2)?(long)*getarg(2):(long)0;
  return (double) nanosleep(&ts,(struct timespec*)0);
ENDVERBATIM
}

:* PROCEDURE spitchar
PROCEDURE spitchar(c) {
VERBATIM
{	
  printf("%c", (int)_lc);
}
ENDVERBATIM
}

:* PROCEDURE spitchar
VERBATIM
static char *pmlc;
ENDVERBATIM

PROCEDURE mymalloc(sz) {
VERBATIM
{ 
  size_t x,y;
  x=(size_t)_lsz;
  pmlc=(char *)malloc(x);
  printf("Did %ld: %x\n",x,pmlc);
  y=(unsigned int)_lsz-1;
  pmlc[y]=(char)97;
  printf("WRITE/READ 'a': "); 
  printf("%c\n",pmlc[y]);
  if (ifarg(2)) free(pmlc); else printf("Use unmalloc() to free memory\n");
}
ENDVERBATIM
}

PROCEDURE unmalloc() {
VERBATIM
  free(pmlc);
ENDVERBATIM
}

:* FUNCTION hocgetc
FUNCTION hocgetc() {
VERBATIM
{	
  FILE* f = hoc_obj_file_arg(1);
  _lhocgetc = (double)getc(f);
}
ENDVERBATIM
}

PROCEDURE pwd() {
  VERBATIM
  {char cwd[1000],cmd[1200];
  getcwd(cwd, 1000);
  sprintf(cmd, "execute1(\"strdef cwd\")\n");         hoc_oc(cmd);
  sprintf(cmd, "execute1(\"cwd=\\\"%s\\\"\")\n",cwd); hoc_oc(cmd);
  }
  ENDVERBATIM
}