Reinforcement learning of targeted movement (Chadderdon et al. 2012)

 Download zip file   Auto-launch 
Help downloading and running models
Accession:144538
"Sensorimotor control has traditionally been considered from a control theory perspective, without relation to neurobiology. In contrast, here we utilized a spiking-neuron model of motor cortex and trained it to perform a simple movement task, which consisted of rotating a single-joint “forearm” to a target. Learning was based on a reinforcement mechanism analogous to that of the dopamine system. This provided a global reward or punishment signal in response to decreasing or increasing distance from hand to target, respectively. Output was partially driven by Poisson motor babbling, creating stochastic movements that could then be shaped by learning. The virtual forearm consisted of a single segment rotated around an elbow joint, controlled by flexor and extensor muscles. ..."
Reference:
1 . Chadderdon GL, Neymotin SA, Kerr CC, Lytton WW (2012) Reinforcement learning of targeted movement in a spiking neuronal model of motor cortex. PLoS One 7:e47251 [PubMed]
Citations  Citation Browser
Model Information (Click on a link to find other models with that property)
Model Type: Realistic Network;
Brain Region(s)/Organism: Neocortex;
Cell Type(s): Neocortex fast spiking (FS) interneuron; Neocortex spiking regular (RS) neuron; Neocortex spiking low threshold (LTS) neuron;
Channel(s):
Gap Junctions:
Receptor(s): GabaA; AMPA; NMDA;
Gene(s):
Transmitter(s): Dopamine; Gaba; Glutamate;
Simulation Environment: NEURON;
Model Concept(s): Simplified Models; Synaptic Plasticity; Long-term Synaptic Plasticity; Reinforcement Learning; Reward-modulated STDP;
Implementer(s): Neymotin, Sam [Samuel.Neymotin at nki.rfmh.org]; Chadderdon, George [gchadder3 at gmail.com];
Search NeuronDB for information about:  GabaA; AMPA; NMDA; Dopamine; Gaba; Glutamate;
/
arm1d
README
drspk.mod *
infot.mod *
intf6_.mod *
intfsw.mod *
misc.mod *
nstim.mod *
stats.mod *
updown.mod *
vecst.mod *
arm.hoc
basestdp.hoc
col.hoc *
colors.hoc *
declist.hoc *
decmat.hoc *
decnqs.hoc *
decvec.hoc *
default.hoc *
drline.hoc *
filtutils.hoc *
geom.hoc
grvec.hoc *
hinton.hoc *
infot.hoc *
init.hoc
intfsw.hoc *
labels.hoc *
local.hoc *
misc.h *
mosinit.hoc
network.hoc
nload.hoc
nqs.hoc *
nqsnet.hoc *
nrnoc.hoc *
params.hoc
run.hoc
samutils.hoc *
sense.hoc *
setup.hoc *
sim.hoc
simctrl.hoc *
stats.hoc *
stim.hoc
syncode.hoc *
units.hoc *
xgetargs.hoc *
                            
// $Id: misc.h,v 1.38 2011/11/02 15:26:48 billl Exp $
#include <stdint.h> /* uint32_t */
#include <stdlib.h>
#include <math.h>
#include <limits.h> /* contains LONG_MAX */
#include <time.h>
#include <sys/time.h>
#include <float.h>
#include <pthread.h>

#if !defined(t)
  #define _pval pval
#endif

typedef struct LISTVEC {
  int isz;
  Object* pL;
  double** pv;
  unsigned int* plen;
  unsigned int* pbuflen;
} ListVec;

typedef struct BVEC {
 int size;
 int bufsize;
 short *x;
 Object* o;
} bvec;

#define BYTEHEADER int _II__;  char *_IN__; char _OUT__[16]; int BYTESWAP_FLAG=0;
#define BYTESWAP(_X__,_TYPE__) \
    if (BYTESWAP_FLAG == 1) { \
    _IN__ = (char *) &(_X__); \
    for (_II__=0;_II__<sizeof(_TYPE__);_II__++) { \
        _OUT__[_II__] = _IN__[sizeof(_TYPE__)-_II__-1]; } \
    (_X__) = *((_TYPE__ *) &_OUT__); \
    }

#define UNCODE(_X_,_J_,_Y_) {(_Y_)=floor((_X_)/sc[(_J_)])/sc[4]; \
                             (_Y_)=floor(sc[4]*((_Y_)-floor(_Y_))+0.5);}
#define MIN(X,Y) ((X) < (Y) ? (X) : (Y))
#define MAX(X,Y) ((X) > (Y) ? (X) : (Y))

//square root of 2 * PI
#define SQRT2PI 2.5066282746310002416
//ln(2), base e log of 2
#define LG2 0.69314718055994530941723212145818
#define VRRY 200
#define ISVEC(_OB__) (strncmp(hoc_object_name(_OB__),"Vector",6)==0)
#define dmaxuint 4294967295. // for 32 bits

// Andre Fentons cast designations
typedef unsigned char   ui1;    /* one byte unsigned integer */
typedef char        si1;    /* one byte signed integer */
typedef unsigned short  ui2;    /* two byte unsigned integer */
typedef short       si2;    /* two byte signed integer */
typedef unsigned int    ui4;    /* four byte unsigned integer */
typedef int     si4;    /* four byte signed integer */
typedef float       sf4;    /* four byte signed floating point number */
typedef double      sf8;    /* eight byte signed floating point number */

extern double ERR,GET,SET,OK,NOP,ALL,NEG,POS,CHK,NOZ,GTH,GTE,LTH,LTE,EQU;
extern double EQV,EQW,EQX,NEQ,SEQ,RXP,IBE,EBI,IBI,EBE;

extern double *vector_newsize (void* vv, int n);
#ifndef NRN_VERSION_GTEQ_8_2_0
extern double* hoc_pgetarg();
extern void hoc_notify_iv();
extern double hoc_call_func(Symbol*, int narg);
extern FILE* hoc_obj_file_arg(int narg);
extern Object** hoc_objgetarg();
char *gargstr();
char** hoc_pgargstr();
extern void vector_resize();
extern int vector_instance_px();
extern void* vector_arg();
extern double* vector_vec();
extern int vector_buffer_size(void*);
extern void mcell_ran4_init(uint32_t);
extern double mcell_ran4(uint32_t* high, double* x, unsigned int n, double range);
extern int nrn_mlh_gsort(double* vec, int* base_ptr, int total_elems, int (*cmp)(double, double));
extern int ivoc_list_count(Object*);
extern int hoc_is_double_arg(int narg);
extern int hoc_is_str_arg(int narg);
extern int hoc_is_object_arg(int narg);
extern int hoc_is_pdouble_arg(int narg);
extern Symbol *hoc_lookup(const char*);
extern Point_process* ob2pntproc(Object*);

extern char* hoc_object_name(Object*);
extern double nrn_event_queue_stats(double*);
extern Symbol *hoc_get_symbol(char *);
extern void cvode_fadvance(double);
extern void clear_event_queue();
extern Object* ivoc_list_item(Object*, int);
extern int hoc_is_tempobj(int narg);
extern int hoc_is_tempobj_arg(int narg);
extern double hoc_epsilon;
extern int stoprun;
extern void set_seed();
#endif
extern unsigned int hashseed2 (int na, double* x);
extern unsigned int  dcrsz;
extern double       *dcr;
extern double       *dcrset(int);
extern unsigned int  scrsz;
extern unsigned int *scr;
extern unsigned int *scrset(int);
extern unsigned int  iscrsz;
extern int *iscr;
extern int *iscrset(int);
extern double BVBASE;
extern void dshuffle(double* x,int nx);
extern void ishuffle(int* x,int nx);
extern unsigned int valseed;
extern int list_vector_px2 (Object *ob, int i, double** px, void** vv);
extern int list_vector_px3 (Object *ob, int i, double** px, void** vv);
int list_vector_px4(Object *ob, int i, double** px, unsigned int n);
extern int cmpdfn(double a, double b);
extern int openvec(int, double **);
int list_vector_px(Object *ob, int i, double** px);
double *list_vector_resize(Object *ob, int i, int sz);
static void hxe() { hoc_execerror("",0); }
extern void FreeListVec(ListVec** pp);
extern ListVec* AllocListVec(Object* p);
extern ListVec* AllocILV(Object*, int, double *);
void FillListVec(ListVec* p,double dval);
void ListVecResize(ListVec* p,int newsz);

static double sc[6];
static FILE*  testout;

//* in vecst.mod
extern int** getint2D(int rows,int cols);
extern void freeint2D(int*** ppp,int rows);
extern double** getdouble2D(int rows,int cols);
extern void freedouble2D(double*** ppp,int rows);
extern double ismono1 (double *x, int n, int flag);
int uniq2(int n, double *x, double *y, double *z);

//* in stats.mod
double kcorfast(double* input1, double* input2, double* i1d , double* i2d,int n,double* ps);
double Rktau (double* x, double* y, int n); // R version
double kcorfast (double* input1, double* input2, double* i1d , double* i2d,int n,double* ps);