Neural Query System NQS Data-Mining From Within the NEURON Simulator (Lytton 2006)

 Download zip file   Auto-launch 
Help downloading and running models
Accession:97874
NQS is a databasing program with a query command modeled loosely on the SQL select command. Please see the manual NQS.pdf for details of use. An NQS database must be populated with data to be used. This package includes MFP (model fingerprint) which provides an example of NQS use with the model provided in the modeldb folder (see readme for usage).
Reference:
1 . Lytton WW (2006) Neural Query System: Data-mining from within the NEURON simulator. Neuroinformatics 4:163-76 [PubMed]
Model Information (Click on a link to find other models with that property)
Model Type: Neuron or other electrically excitable cell;
Brain Region(s)/Organism:
Cell Type(s):
Channel(s):
Gap Junctions:
Receptor(s):
Gene(s):
Transmitter(s):
Simulation Environment: NEURON;
Model Concept(s): Methods;
Implementer(s): Lytton, William [bill.lytton at downstate.edu];
// $Id: nqs.hoc,v 1.269 2005/03/08 16:56:19 billl Exp $

if (!name_declared("VECST_INSTALLED")) {
  printf("NQS ERROR: Need vecst.mod nmodl package compiled in special.\n")
}
if (!VECST_INSTALLED) install_vecst()
load_file("setup.hoc")
load_file("grvec.hoc")

strdef execstr,strform,dblform
{strform="%s " dblform="%3.4g "}

//* stubs for ancillary programs
double sops[19]  // AUGMENT TO ADD NEW OPSYM
proc sopset () {}
func whvarg () {}
proc whkey () {}
func varstr () {}

//* NQS template
// potential to overwrite XO,tmpfile,i1
begintemplate NQS
public cob,out,up // operate on this or out 
public s,comment,file,v,m,x,ind,scr,fcd,fcds,fcdo,fcdl,this,sstr // strings and vecs
public objl,verbose,tmplist,vlist,nval,sval,oval,selcp,rxpstr,slorflag,stub
public sv,rd,append,pr,prn,zvec,resize,size,fi,sets,set,gets,get,fetch,tog   // routines
public cp,mo,aind,it,qt,ot,appi,eq,fcdseq,sort,select,stat,rdcols,map,apply,applf,remove
public spr,pad,delect,fill,uniq,gr,clear,strdec,join,jn,fillin,fillv,useslist,otl,selall
objref v[1],s[1],is[3],x,nil,ind,scr[3],fcd,fcds,fcdo,fcdl,this,objl
objref cob,out,up,Xo,Yo,oval,tmplist,otl,vlist

strdef comment,file,sstr,sstr2,sstr3,sstr4,tstr,sval
double m[1]
external readnums,savenums,readdbls,savedbls,rdvstr,wrvstr,sfunc,repl_mstr,isobj
external vlk,Union,String,tmpfile,strm,XO,execstr,i1,allocvecs,dealloc,mso,strform,dblform
external eqobj,isnum,chop,isassigned,whvarg,whkey,sops,batch_flag,g,varstr

//** init()
proc init () { local i,ii,flag,scnt,na,fl
  nval=fl=scnt=flag=0 // flag set if creating the internal NQS
  selcp=verbose=1
  for ii=1,2 is[ii]=new String()
  is[1].s="INDEX"  is[2].s="SCRATCH"
  na=numarg()
  for i=1,na scnt+=(argtype(i)==2) // string count
  if (na==0) scnt=-1
  if (na==1) if (argtype(1)==2) { rd($s1) return }
  if (na>=1) if (argtype(1)==0) {
    fl=1 // 1 arg taken care of
    if ($1==1e-9) { flag=1 } else {
      m=$1
      objref v[m],s[m]
      for ii=0,m-1 { v[ii]=new Vector() s[ii]=new String2() }
    }
  }
  if (fl!=1 && na==scnt) { // all strings
    fl=2 // all args taken care of
    m=na
    objref v[m],s[m]
    for ii=0,m-1 {i=ii+1 v[ii]=new Vector() s[ii]=new String($si) }
  }
  if (fl!=2 && na>=2) if (argtype(2)==0) { 
    fl==2  // all args taken care of
    for ii=0,m-1 v[ii].resize($2) 
  }
  if (fl!=2) { // if first arg is not a string these other can be
    if (na>=2) file=$s2
    if (na>=3) comment=$s3
    if (na>=4) x.x[0]=$4
  }
  if (!flag) { 
    // fcd gives field codes according to values used for argtype()
    fcds=new List() fcd=new Vector(m) tmplist=new List() vlist=new List()
    fcd.resize(m) fcd.fill(0) // field codes to have a field that's string based
  }
  x=new Vector(m) ind=x.c for ii=0,2 scr[ii]=x.c
  scr.resize(0) ind.resize(0)
  objl=new List() cob=this
  slorflag=0
  if (!flag) {out=new NQS(1e-9) out.up=this out.cp(this,0)}
}

//** tog() toggle flag that determines whether actions are on out or this
proc tog () { 
  if (numarg()==0) {
    if (eqobj(cob,out)) { cob=this print "Operate on full db"
    } else {              cob=out  print "Operate on output of select" }
  } else if (numarg()==1) {
    if (argtype(1)==0) {  // just give information
      if (eqobj(cob,out)) { print "Using output db"
      } else {              print "Using full db" }
    } else if (argtype(1)==2) { // out,output,selected to choose these
      if (strm($s1,"[Oo][Uu][Tt]") || strm($s1,"[Ss][Ee][Ll]")) {
        cob=out } else cob=this
   }
  }
}

//** sets() set the strings to given args
proc sets () { local i,nm
  nm=numarg()
  if (nm==2 && argtype(1)==0) s[$1].s=$s2 else {
    if (nm>m) { 
      if (batch_flag) {
        printf("NQS sets WARNING resized table from %d to %d\n",m,nm)
      } else if (! boolean_dialog("Resize TABLE?","YES","NO")) return
      printf("resizing TABLE: %d -> %d\n",m,nm) resize(nm) 
    }
    for i=1,nm { s[i-1].s=$si out.s[i-1].s=$si }
  }
}
// gets() print the strings
proc gets () { for ii=0,m-1 printf("%s(%d) ",s[ii].s,ii) }

//* select() -- based loosely on SQL select
func select () { local ii,i,ret,isv,key,arg,vc,selcpsav,savind,union,not,rxpflg
  if (numarg()==0) { out.cp(this,2) cob=out return v.size }
  if (size(1)==-1) { printf("NQS:select ERR0: cols not all same size\n") return -1 }
  // key holds OPs; arg holds ARGs; vc holds COL NAMEs
  key=arg=vc=allocvecs(3) arg+=1 vc+=2
  selcpsav=selcp i=1 not=rxpflg=union=savind=0
  tmplist.remove_all vlist.remove_all
  tog("DB") // start at full db
  if (argtype(i)==0) { i+=1
    if ($1==-1) { selcp=0 } else { 
      printf("NQS:select ERRa: first vec obj should be ind vector\n") dealloc(key) return -1 }
  }
  if (argtype(i)==2) { // check first string for &&, ||, !
    if (strcmp($si,"&&")==0) {        savind=1 union=0 i+=1 
    } else if (strcmp($si,"||")==0) { savind=1 union=1 i+=1 
    } else if (strcmp($si,"!")==0)  { savind=0  not=1 i+=1
    } else if (strcmp($si,"&&!")==0)  {savind=1 not=1 i+=1
    } else if (strcmp($si,"||!")==0)  {savind=1 union=1 not=1 i+=1 }
  } else if (argtype(i)==1) { i+=1
    if (isobj($o1,"Vector")) { ind.copy($o1) savind=1 union=0 // assume &&
    } else { 
      printf("NQS:select ERR0a: first vec obj should be ind vector\n") dealloc(key) return -1 }
  }
  if (savind) scr.copy(ind) else scr.resize(0)

  for (;i<=numarg();) {
    if (argtype(i)==2) { 
      if (strcmp($si,"INDEX")==0) {
        if ((vn=fi($si,"NOERR"))!=-1) { 
          printf("NQS:select() WARNING: INDEX is a reserved word: ?%s\n",s[vn].s) }
        vn=-1e9  scr[1].indgen(0,v.size-1,1) tmplist.prepend(scr[1])
      } else if ((vn=fi($si))<0) { dealloc(key) return -1 }
      sstr=$si  // save for join: use with "NAME",EQW,OTHER_NQS
    } else if (argtype(i)==0) { vn=$i // can avoid repeated string search
      sstr=s[vn].s
    } else {printf("NQS:select ERR1: arg %d should be col name or num\n",i) dealloc(key) return -1}
    mso[vc].append(vn)                        i+=1 
    if (argtype(i)==0) {
      if ((isv=isvarg($i))==-1) { 
        mso[key].append(EQU) // if arg2 is a regular number presume that op is EQU arg2
        mso[arg].append($i,0)
        i+=1
        continue
      } else { lk=$i }
    } else if (argtype(i)==2) { isv=isvarg(lk=whvarg($si))
      if (isv==-1) { printf("NQS:select ERR1a: operator %s not recognized\n",$si) dealloc(key) return -1 }
    } else {
      printf("NQS:select ERR2: arg should be symbolic (eg GTE, EQU ...) or string (eg '[)','<=') op \n",i)
      dealloc(key) return -1 
    }
    mso[key].append(lk)                       i+=1
    // pick up ARGS
    for ii=0,isv-1   {  
      if (argtype(i)==0) { 
        mso[arg].append($i)                   i+=1 
      } else if (argtype(i)==2) {
        if (lk==EQV) { // look for a column id
          vn=fi($si) // OPSYM exception
          if (vn==-1) { printf("NQS:select ERR2a EQV but what col?\n") dealloc(key) return -1 }
          mso[arg].append(0)
          mso[vc].append(vn)                    i+=1
        } else if (lk==SEQ) {
          mso[key].x[mso[key].size-1]=EQU
          if (argtype(i)==1) oval=$oi else if (argtype(i)==2) sval=$si
          mso[arg].append(ret=finval(vn,argtype(i),lk))  i+=1
          if (ret==ERR) return -1
        } else if (lk==RXP) {
          mso[key].x[mso[key].size-1]=EQW
          mso[arg].append(0)
          if (argtype(i)!=2) {printf("NQS:select ERR2a1\n") dealloc(key) return -1}
          if (rxpflg==1) {printf("NQS:select ERR2a2: RXP twice\n") dealloc(key) return -1}
          ret=tmplist.prepend(scr[2]) rxpflag=1
          if (rxpstr(vn,$si,scr[2])==0) {
            printf("NQS:select WARNING: No RXP matches for %s\n",$si) }
          mso[vc].append(-1e9)                  i+=1
        } else {printf("NQS:select ERR2b string arg needs EQV,SEQ or RXP?\n")
          dealloc(key) return -1}
      } else if (argtype(i)==1) { 
        if (lk==EQW) { // pick up a vector
          if (isobj($oi,"Vector")) {
            mso[arg].append(0)
            mso[vc].append(-i)                    i+=1
          } else if (isobj($oi,"NQS")) {
            mso[arg].append(0)
            tmplist.prepend($oi.out.v[$oi.fi(sstr)]) // assume JOIN with output from other nqs
            mso[vc].append(-1e9)                  i+=1
          } else { printf("NQS:select ERR2c: EQW needs Vec or NQS not %s?\n",$oi)
            dealloc(key) return -1
          }
        } else { printf("NQS:select ERR2d only EQW takes obj arg: %d:%d?\n",i,argtype(i))
          dealloc(key) return -1}
      } else {
        whkey(lk,sstr) printf("NQS:select ERR3 arg %d should be arg for %s",i,sstr) 
        dealloc(key) return -1
      }
    }
    // ERR for args in wrong order
    if (isv==2) if (mso[arg].x[mso[arg].size-2]>mso[arg].x[mso[arg].size-1]) {
      whkey(lk,sstr)
      printf("NQS:select ERR4 2 args for %s are in wrong order: %g %g\n",\
             sstr,mso[arg].x[mso[arg].size-2],mso[arg].x[mso[arg].size-1])
      dealloc(key) return -1      
    }
    // pad so every OP sees 2 ARGS
    for ii=0,2-isv-1 { mso[arg].append(0) } 
  }
  ind.resize(v.size)

  for ii=0,mso[vc].size-1 { vn=mso[vc].x[ii] 
    if (vn==-1e9) { // code for EQW case with NQS arg
      vlist.append(tmplist.object(tmplist.count-1)) 
      tmplist.remove(tmplist.count-1) // pop
    } else if (vn<0) { i=-vn // code for EQV case where vector is in the arg list
      vlist.append($oi) 
    } else vlist.append(v[vn]) 
  }
  if (tmplist.count!=0) { printf("NQS:select ERR5 %s.tmplist not empty\n",this) return -1 }

  if (slorflag) { ind.slor(mso[key],mso[arg],vlist)
  } else        { ind.slct(mso[key],mso[arg],vlist) }

  if (not==1) complement() // ind->!ind
  if (savind) { 
    if (union==1) {
      scr.append(ind) scr.sort ind.resize(scr.size+ind.size)
      ind.redundout(scr)
    } else {
      mso[key].resize(scr.size+ind.size)
      mso[key].insct(scr,ind) ind.copy(mso[key]) }
  }
  ret=ind.size
  if (selcp) {
    out.ind.copy(ind) 
    aind()
    cob=out
  } else cob=this
  dealloc(key)
  selcp=selcpsav
  return ret
}

//** selall()
proc selall () { local ii
  if (numarg()==2) {
    for ii=0,m-1 out.v[ii].where(v[ii],$s1,$2)
  } else {
    for ii=0,m-1 out.v[ii].where(v[ii],$s1,$2,$3)
  }
  tog("SEL")
}

//** complement() ind -> !ind
proc complement () { local a,b
  a=b=allocvecs(2) b+=1
  mso[a].indgen(0,size(1)-1,1)
  mso[b].resize(mso[a].size)
  mso[b].cull(mso[a],ind)
  ind.copy(mso[b])
  dealloc(a)
}

//** delect([NQS])
// move the selected rows from the out db [or other] back to the main db
// the assumption is that you have operated on some of the fields and now want to
//      put the rows back
// ind must not have been altered since it will be used to replace the items
func delect () { local beg,ii,flag
  scr.resize(v.size)
  if (numarg()==1) flag=1 else flag=0
  if (flag) { 
    if (m!=$o1.m){ 
      printf("NQS:delect ERRa m mismatch: %s:%d vs %s:%d\n",this,m,$o1,$o1.m) return -1 }
    ind.copy($o1.ind)
  } else if (!out.ind.eq(ind) || ind.size!=out.v.size) {
    printf("NQS:delect ERR ind size mismatch\n") 
    return -1 
  }
  for (beg=0;beg<m;beg+=11) { // sindx() can only handle 11 vecs at a time
    tmplist.remove_all vlist.remove_all
    for ii=beg,beg+10 if (ii<m) tmplist.append(v[ii])
    for ii=beg,beg+10 if (ii<m) if (flag) { 
      vlist.append($o1.v[ii]) 
    } else {
      vlist.append(out.v[ii]) 
    }
    ind.sindx(tmplist,vlist)
  }      
  cob=this
  return ind.size
}  
  
//** isvarg() returns number of args an op takes or -1 if not symbolic OP
func isvarg () { // ADD NEW OPSYM CHECK
  if ($1<ALL) return -1 else if ($1<GTH) return 0 else if ($1<IBE) { return 1 
  } else if ($1<=EBE) return 2 else return -1 
}

//** fi(STR[,XO]) find the index for a particular string, can set a objref
//  fi(STR,INDEX) return INDEXed value from that vector
//  fi(STR,-1) suppress error message
func fi () { local num,flag,ii,ret,err
  noerr=err=num=flag=0
  if (numarg()==2) if (argtype(2)==2) noerr=1 // use "NOERR" string
  for ii=0,m-1 if (strcmp(s[ii].s,$s1)==0) {flag=1 ret=ii break} // exact match
  if (strcmp($s1,"scr")==0 || strcmp($s1,"SCR")==0) {flag=1 ret=-2}
  if (strcmp($s1,"INDEX")==0) {flag=1 ret=-3}
  if (!flag) for ii=0,m-1 if (strm(s[ii].s,$s1)) { 
    if (num>=1) {
      err=1 printf("NQS fi ERR: regexp matches more than once: %d %s\n",ii,s[ii].s)
    } else {
      num+=1 ret=ii flag=1
    }
  }
  if (err) printf("NQS WARNING; ambiguous regexp; fi() returning pointer for: %d %s\n",ret,s[ret].s)
  if (flag) {
    if (numarg()==2 && noerr==0) { 
      if        (argtype(2)==1) { 
        if (ret==-2) $o2=scr else if (ret==-3) {printf("NQS:fi ERRa copy what?\n") return ret
        } else $o2=v[ret] 
      } else if (argtype(2)==0) { 
        if ($2<0 || $2>=v[ret].size) { 
          printf("NQS:fi ERR index out of bounds: %d %d\n",$2,v[ret].size) 
          return -1
        }
        if (ret==-2) ret=scr.x[$2] else if (ret==-3) {printf("NQS:fi ERRb what?\n") return ret
        } else ret=v[ret].x[$2]
      } else                    { printf("NQS:fi WARNING 2nd arg ignored\n") }
    }
    return ret
  } else {
    if (!noerr) printf("NQS.fi() ERR '%s' not found\n",$s1)
    return -1
  }
}

//** set("name",IND,VAL)
proc set () { local fl,ix,i
  if (eqobj(cob,out) && verbose) printf("Selected: ") 
  fl=fi($s1) ix=$2 i=3
  if (fl==-1) return
  // 2 LINE 'SET' MACRO
  if (argtype(i)==0) nval=$i else if (argtype(i)==1) oval=$oi else if (argtype(i)==2) sval=$si
  cob.v[fl].x[ix]=newval(argtype(i),fl)
}

//** newval(typ,col#) -- check if a value is already on the list and if not put it there
// usuall preceded by eg:
// if (argtype(i)==0) nval=$i else if (argtype(i)==1) oval=$oi else if (argtype(i)==2) sval=$si
func newval () { local ret,typ,flag,fl,ii
  typ=$1 fl=$2
  if (fcd.x[fl]!=typ && fcd.x[fl]!=-1) { 
    printf("nqs::newval() ERRa %d statt %d\n",typ,fcd.x[fl]) return ERR }
  if (typ==0 || typ==-1) { 
    return nval
  } else if (typ==1) { // object handling
  } else if (typ==2) { // string handling
    for (ii=flag=0;ii<fcds.count;ii+=1) { 
      Xo=fcds.object(ii)
      if (strcmp(Xo.s,sval)==0) {flag=1 return ii}
    }
    if (!flag) return fcds.append(new String(sval))-1
  }
}

//*** finval(col#,type,OP) find the location on list for an object or string
func finval () { local fl,typ,op,ii,ret
  fl=$1 typ=$2 op=$3 ret=-1
  if (fcd.x[fl]!=typ) { // doesn't handle fcd.x[]==-1
    printf("nqs::finval ERRa type mismatch; %d %d\n",fcd.x[fl],typ) return ERR }
  for ii=0,fcds.count-1 { Xo=fcds.object(ii)
    if (typ==2) { 
      if (strcmp(Xo.s,sval)==0) return ii
    } else {}
  }
  if (ret>-1) return ret
  printf("nqs::finval ERRc %s not found in string or object list\n",sval) 
  return ERR
}

//*** rxpstr(col#,vec) find the location on list for an object
func rxpstr () { local fl
  fl=$1 $o3.resize(0)
  if (fcd.x[fl]!=2) {
    printf("nqs::rxpstr ERRa type mismatch; %d %d\n",fcd.x[fl],2) return -1 }
  for ii=0,fcds.count-1 if (strm(fcds.object(ii).s,$s2)) $o3.append(ii)
  return $o3.size
}

//*** getval(col#,index) return type and value in nval,oval,sval as appropriate
// usually followed by eg
// if (typ==0) ... nval else if (typ==1) ... oval else if (typ==2) ... sval
func getval () { local typ,n,flag,fl,ix,ii
  fl=$1 ix=$2 flag=0
  typ=fcd.x[fl] // argtype
  if (typ==0) { 
    nval=ix
  } else if (typ==1) { // object handling
    // oval = ...
  } else if (typ==2) { // string handling
    if (ix==-1) {
      sval="NULL"
    } else if (ix<0 || ix>fcds.count-1) {
      printf("nqs::getval() ERR index OOB %d, %d\n",ix,fcds.count) return ERR 
    } else sval=fcds.object(ix).s
  } else if (typ==-1) { // string from external list
    if (fcdl.count<=fl) {printf("NQS getval ERRa\n") return -1}
    if (! isobj(fcdl.object(fl),"List")) {printf("NQS getval ERRb\n") return -1}
    if (fcdl.object(fl).count<=ix) {printf("NQS getval ERRc\n") return -1}
    if (ix==-1) sval="XX" else {
      if (! isobj (fcdl.object(fl).object(ix),"String")) {printf("NQS getval ERRd\n") return -1}
      sval=fcdl.object(fl).object(ix).s
    }
  }
  return typ
}

//*** useslist() connects a list of strings to fcdl to use when printing
// fcdl: list of lists to make it easy to attach lists from outside
proc useslist () { local fl,ii
  if (argtype(1)==2) fl=fi($s1) else fl=$1
  if (fl==-1) return
  if (! isobj(fcdl,"List")) {fcdl=new List() out.fcdl=fcdl}
  for ii=fcdl.count,m-1 fcdl.append(fcdl) // use fcdl as placeholder
  fcdl.remove(fl) fcdl.insrt(fl,$o2)  // replace:fcdl.object(fl)=$o2
  fcd.x[fl]=-1
}

//*** prtval() use %g or %s to print values
proc prtval () { local typ,flag
  typ=$1
  if (typ==0) sstr=dblform else sstr=strform
  if (numarg()==2) sprint(sstr,"%s%s",sstr,$s2)
  if (numarg()==3) sprint(sstr,"%s%s%s",$s2,sstr,$s3)
         if (typ==0) { printf(sstr,nval)
  } else if (typ==1) { printf(sstr,oval) 
  } else if (typ==2) { printf(sstr,sval) 
  } else if (typ==-1) { printf(sstr,sval) } // special code for externally provided list
}

//** get("name",[IND]]) if omit IND take ind from first ind.x[0]
obfunc get () { local ty,fl,ix,outf localobj lo
  outf=0
  if (argtype(1)==0) { fl=$1 sstr2=s[fl].s
  } else if (argtype(1)==2) { fl=fi($s1) sstr2=$s1 }
  if (fl==-1) { return -1 }
  if (eqobj(cob,out)) { outf=1 
    if (verbose) printf("Selected: ") }
  if (numarg()==1) {
    if (outf) ix=0 else ix=ind.x[0]
  } else ix=$2
  if (ix<0 || ix>=cob.v[fl].size) {
    printf("NQS::get ERR ix %d out of range for %s (%s)\n",ix,sstr2,cob) return -1 }
  ty=fcd.x[fl]
  if (ty==0) lo=new Union(cob.v[fl].x[ix])
  if (ty==2) lo=new Union(fcds,cob.v[fl].x[ix])
  if (ty==-1) lo=new Union(fcdl.object(fl),cob.v[fl].x[ix])
  return lo
}

//** fetch(COLA,VAL,COLB) does fast select where COLA is VAL and returns value in COLB
// only works with VAL as number
func fetch () { local fl1,fl2
  if (argtype(1)==2) fl1=fi($s1) else fl1=$1
  if (argtype(3)==2) fl2=fi($s3) else fl2=$3
  return v[fl2].x[v[fl1].indwhere("==",$2)]
}

//** stat("name","vec_operation")
proc stat () { local i,vn
  if (eqobj(cob,out) && verbose) printf("Selected: ") 
  if (argtype(1)==0) vn=$1 else vn=fi($s1)
  if (vn<0||vn>=m) return
  if (cob.size(1)==0) { print "NQS:stat Empty NQS" return }
  if (vn==-2) sprint(sstr2,"%s",scr) else sprint(sstr2,"%s",cob.v[vn])
  if (numarg()==1) {
    sprint(sstr,   "printf(\"max=%%g; \",%s.max) ",sstr2)
    sprint(sstr,"%s printf(\"min=%%g; \",%s.min) ",sstr,sstr2)
    sprint(sstr,"%s printf(\"mean=%%g; \",%s.mean) ",sstr,sstr2)
    sprint(sstr,"%s printf(\"stdev=%%g; \",%s.stdev) ",sstr,sstr2)    
    execute(sstr)
  } else for i=2,numarg() {
    if (strm($si,"[(][)]$")) {
      sfunc.left($si,sfunc.len($si)-2)
      sprint(sstr,"printf(\"%s()=%%g; \",%s(%s))",$si,$si,sstr2)
    } else sprint(sstr,"printf(\"%s=%%g; \",%s.%s)",$si,sstr2,$si)
    execute(sstr)
  }
  print ""
}

//** iterator it() set's global tstr and XO to string bzw vec
iterator it () { local ii
  i1=0
  for ii=0,m-1 {
    XO=cob.v[ii] execstr=s[ii].s
    iterator_statement
    i1+=1
  }
}

//** iterator ot() creates names for each col (same as col header) and goes through them all
iterator ot () { local i,na,val
  if (! isobj(otl,"List")) { // create list to execute
    otl=new List()
    for i=0,m-1 { 
      Xo=new String(s[i].s)
      if (fcd.x[i]==2) {
        varstr(Xo.s,1) 
        sprint(Xo.s,"%s=%s.fcds.object(%s.cob.v[%d].x[i1]).s",Xo.s,this,this,i)
      } else {
        varstr(Xo.s) 
        sprint(Xo.s,"%s=%s.cob.v[%d].x[i1]",Xo.s,this,i)
      }
      otl.append(Xo)
    }
    Xo=nil
  }
  for (i1=0;i1<cob.v[0].size;i1+=1) {
    for i=0,m-1 execute(otl.object(i).s)
    iterator_statement
  }
}

//** iterator qt(&x1,NAME1,&x2,NAME2,...) 
// eg for sp.qt(&x,"PRID",&y,"POID",&z,"NC1",&ii,"WID1",&jj,"WT1") print x,y,z,ii,jj
iterator qt () { local i,ii,na,val
  na=numarg() scr.resize(0)
  if (na/2!=int(na/2)) {print "NQS::qt() needs even # of args\n" return }
  if (eqobj(cob,out) && verbose) printf("Selected: ") 
  for (i=2;i<=na;i+=2) { 
    if (argtype(i)!=2) {printf("NQS::qt() ERR arg %d should be col name\n",i) return }
    scr.append(val=fi($si))
    if (val==-1) {printf("NQS::qt() ERR %s not found\n",$si) return }}
  scr[1].copy(fcd)
  for (i=1;i<=na;i+=2) {
    // can't do iteration over externally defined strings (eg -1) see useslist()
    if (scr[1].x[scr.x[int(i/2)]]!=0) { 
      if (argtype(i)!=2) {
        printf("NQS::qt() WARNING using list index statt str for col %s\n",s[scr.x[int(i/2)]].s) 
        if (argtype(i)!=3) {printf("NQS::qt() ERR arg %d should be pointer\n",i) return}
        scr[1].set(scr.x[int(i/2)],0)
      }
    } else if (argtype(i)!=3) {
      printf("NQS::qt() ERR arg %d should be pointer\n",i) return }
  }
  for (i1=0;i1<cob.v[0].size;i1+=1) {
    for (i=1;i<=na;i+=2) if (scr[1].x[scr.x[int(i/2)]]==0) {
           $&i=            cob.v[scr.x[int(i/2)]].x[i1]
    } else $si=fcds.object(cob.v[scr.x[int(i/2)]].x[i1]).s
    iterator_statement
    for (i=1;i<=na;i+=2) if (scr[1].x[scr.x[int(i/2)]]==0) {
      cob.v[scr.x[int(i/2)]].x[i1]=$&i
    } else {
      fcds.object(cob.v[scr.x[int(i/2)]].x[i1]).s=$si
    }
  }
}

//** spr() spread-sheet functionality using vector functions
// takes a compound expression utilizing column names in slant brackets <>
// anything quoted can use either ' or \"
// eg sp.spr("<DIST>.c.mul(DELD).add(DEL)")
proc spr () { local ii,vn
  if (numarg()==0) { 
    printf("eg spr(\"<SCR>.copy(<COL1>.c.mul(<COL2>).add(5))\") \ntakes a compound expression utilizing column names in slant brackets <>\nanything quoted can use either ' slash quote.\n")
    return 
  } else sstr=$s1
  if (eqobj(cob,out) && verbose) printf("Selected: ") 
  while (sfunc.tail(sstr,"<",sstr2)!=-1) {
    sfunc.head(sstr2,">",sstr2)
    if (strcmp(sstr2,"SCR")==0) { 
      sprint(sstr3,"%s",cob.scr)
    } else if ((vn=fi(sstr2))==-1) return else {
      sprint(sstr3,"%s",cob.v[vn])      
    }
    sprint(sstr2,"<%s>",sstr2)
    repl_mstr(sstr,sstr2,sstr3,sstr4)
  }
  repl_mstr(sstr,"'","\"",sstr4)
  execute(sstr)
}

//** sort () sort according to one index
func sort () { local beg,ii,vn
  if (eqobj(cob,out) && verbose) printf("Selected: ") 
  if ((vn=fi($s1))<0) return -1
  cob.v[vn].sortindex(cob.ind)
  if (numarg()==2) if ($2==-1) cob.ind.reverse
  cob.scr.resize(cob.v.size)
  for (beg=0;beg<m;beg+=10) { // fewind() can only handle 10 vecs at a time
    tmplist.remove_all
    for ii=beg,beg+9 if (ii<m) tmplist.append(cob.v[ii])
    cob.scr.fewind(cob.ind,tmplist)
  }      
  cob.ind.resize(0) // prevents reusing it
  return vn
}  

//** uniq(COLNAME) will pick out unique row (1st) for the chosen column
func uniq () { local vn
  if (! eqobj(cob,out)) {printf("Only run NQS:unq() on prior selected set.\n") return}
  vn=sort($s1)
  cob.ind.resize(cob.v.size)
  cob.ind.redundout(cob.v[vn],1)
  for (beg=0;beg<m;beg+=10) { // fewind() can only handle 10 vecs at a time
    tmplist.remove_all
    for ii=beg,beg+9 if (ii<m) tmplist.append(cob.v[ii])
    cob.scr.fewind(cob.ind,tmplist)
  }      
  cob.ind.resize(0) // prevents reusing it
  return vn
}  

//** remove() will remove a row
proc remove () { local ii,ix
  ix=$1
  for ii=0,m-1 v[ii].remove(ix)
}

//** aind () -- index all of the vecs
proc aind () { local beg,ii
  for (beg=0;beg<m;beg+=10) {
    tmplist.remove_all vlist.remove_all
    for ii=beg,beg+10 if (ii<m) {
      out.v[ii].resize(ind.size)
      tmplist.append(v[ii])
      vlist.append(out.v[ii])
    }
    ind.findx(tmplist,vlist)
  }      
}  

//** append(VEC) or append(x1,x2,...) appends to ends of given vectors
proc append () { local ii,i,flag
  cob=this
  if (numarg()==1 && argtype(1)==1) {
    if (isobj($o1,"Vector")) { // a vector
      if ($o1.size>m) { print "NQS append ERR1: vec too large; doing nothing"
      } else {
        for i=0,$o1.size-1 v[i].append($o1.x[i])
      }
    } else if (isobj($o1,"NQS")) { // another NQS to add onto end
      if ($o1.m != m) { 
        printf("NQS append ERR1a, %s size (%d)!= %s size (%d)what is %s?\n",this,m,$o1,$o1.m)
        return }
      for ii=0,m-1 v[ii].append($o1.v[ii])
      ind.append($o1.ind)
    } else { printf("NQS append ERR1b, what is %s?\n",$o1) return }
  } else if (numarg()>m) { print "NQS append ERR2: args>m; doing nothing" return
  } else if (numarg()==m) {
    for ii=0,m-1 {
      i=ii+1
      if (argtype(i)==0) nval=$i else if (argtype(i)==1) oval=$oi else if (argtype(i)==2) sval=$si
      v[ii].append(newval(argtype(i),ii))
    }
  } else if (argtype(1)==2) {
    for i=1,numarg() { 
      if ((ii=fi($si))==-1) return  
      i+=1 
      if (argtype(i)==0) nval=$i else if (argtype(i)==1) oval=$oi else if (argtype(i)==2) sval=$si
      v[ii].append(newval(argtype(i),ii))
    }
  }
}

//** appi(index,VEC) or append(index,x1,x2,...) appends to ends of vectors starting at index
proc appi () { local i,flag,ix
  flag=0 cob=this ix=$1
  if (numarg()==2 && argtype(2)==1) { // a vector
    if ($o1.size>m-ix) { print "NQS appi ERR1: vec too large; doing nothing"
    } else {
      for i=ix,$o1.size-1 v[i].append($o1.x[i])
    }
  } else {
    if (numarg()-1>m-ix) {
      print "NQS appi ERR2: args>m; doing nothing"
      flag=1
    } 
    if (! flag) for i=2,numarg() v[ix+i-2].append($i)
  }
}

//** map(FUNC,arg1,...) map $s1 command to other args, replacing strings with vectors as found
// eg nqs.map("gg",0,"volt","cai",2)
proc map () { local i,agt,wf
  if (numarg()==0) { 
    printf("map(FUNC,arg1,...) apply function to args using names for columns.\n")
    return }
  if (eqobj(cob,out) && verbose) printf("Selected: ") 
  sprint(sstr,"%s(",$s1) // the command
  wf=0
  for i=2,numarg() { // the args
    agt=argtype(i)
    if (agt==0) {
      sprint(sstr,"%s%g,",sstr,$i)
    } else if (agt==1) {
      sprint(sstr,"%s%s,",sstr,$oi)
    } else if (agt==2) {
      if ((vn=fi($si))==-1) {
        sprint(sstr,"%s\"%s\",",sstr,$si) 
        printf("NQS.map WARNING: including raw string: %s\n",$si) wf=1
      } else if (vn==-2) { // code for scr vector
        sprint(sstr,"%s%s,",sstr,cob.scr) 
      } else {
        sprint(sstr,"%s%s,",sstr,cob.v[vn]) 
      }
    } else { printf("argtype %d for arg %d not implemented for NQS:map\n",agt,i) return }
  }
  chop(sstr) sprint(sstr,"%s)",sstr)
  if (wf && !batch_flag) if (boolean_dialog(sstr,"CANCEL","EXECUTE")) return
  execute(sstr)
}

//*** gr() use map to graph
// need to assign .crosshair_action so can do visual select procedure
proc gr () { local nm,ii
  nm=numarg() ii=0
  if (nm==0) { print "gr(\"Y\"[,\"X\",g#,,col,line])" return
  } else if (nm==1) {  map("gg",0,$s1,1) 
  } else if (nm==2) {  map("gg",0,$s1,$s2)
  } else if (nm==3) {  map("gg",$3,$s1,$s2) ii=$3
  } else if (nm==4) {  map("gg",$3,$s1,$s2,$4) ii=$3 g[ii].color($4)
  } else if (nm==5) {  map("gg",$3,$s1,$s2,$4,$5) ii=$3 g[ii].color($4)
  }
  g[ii].label(0.05,0.95,$s1)
  if (nm>=2) g[ii].label(0.95,0.05,$s2)
  g[ii].color(1)
}

//** apply function or .op to every selected vector -- ignore return val, see applf
proc apply () { local i,fl
  if (numarg()==0) { 
    printf("apply(FUNC,COL1,...) apply function or .op to every selected vector.\n")
    printf("must be function, not proc, since will return value.\n")
    return }
  if (numarg()==1) for i=0,m-1 { // apply to all vectors
    if (strm($s1,"^\\.")) sprint(sstr,"%s%s"  ,cob.v[i],$s1) else {
                          sprint(sstr,"%s(%s)",$s1,cob.v[i]) }
    execute(sstr)
  } else for i=2,numarg() {
    if ((fl=fi($si))==-1) return
    if (strm($s1,"^\\.")) sprint(sstr,"%s%s"  ,cob.v[fl],$s1) else {
                          sprint(sstr,"%s(%s)",$s1,cob.v[fl]) }
    execute(sstr)
  }
}

//** applf() function or .op which returns a value
func applf () { local fl
  if (numarg()==0) { 
    printf("apply(FUNC,COLNAME) apply function or .op to selected vector.\n")
    printf("must be function, not proc, since will return value.\n")
    return -1 }
  if ((fl=fi($s2))==-1) return -1
  if (strm($s1,"^\\.")) sprint(sstr,"i1=%s%s"  ,cob.v[fl],$s1) else {
                        sprint(sstr,"i1=%s(%s)",$s1,cob.v[fl]) }
  execute(sstr)
  return i1
}

//** fill(NAME,val[,NAME1,val1 ...])
// fill each selected vector with next arg
proc fill () { local i,fl,fl2
  if (numarg()==0) { 
    printf("fill(NAME,val[,NAME1,val1 ...])\n\tfill each selected vector with val\nval can be num, vector, or other col name\n")
    return }
  for i=1,numarg() { fl=fi($si) i+=1
    if (argtype(i)==0) {
      if (fl>-1) cob.v[fl].fill($i)
    } else if (argtype(i)==1) {
      if (!isobj($oi,"Vector")){printf("NQS:fill() ERRa: only fill with vector: %s\n",$oi) return}
      if ($oi.size!=cob.v.size){
        printf("NQS:fill() ERRb: wrong vec size: %d!=%s:%d\n",cob.v.size,$oi,$oi.size) return}
      cob.v[fl].copy($oi)
    } else if (argtype(i)==2) {
      fl2=fi($si)
      if (fl2>-1) cob.v[fl].copy(cob.v[fl2])
    }
  }
}

//** fillin(NAME,val[,NAME1,val1 ...])
// fill in place according to indices in ind -- use with selcp=0
proc fillin () { local i,fl
  if (numarg()==0) { 
    printf("fillin(NAME,val[,NAME1,val1 ...])\n\tfill selected vectors in place\n")
    printf("\tuse after select(-1,...) eg selcp==0\n")
    return 
  }
  scr.resize(0)
  for (i=2;i<=numarg();i+=2) scr.append($i)
  tmplist.remove_all
  for (i=1;i<=numarg();i+=2) {
    if (argtype(i)==2) { 
      if ((fl=fi($si))==-1) return
    } else fl=$i
    tmplist.append(v[fl])
  }
  ind.sindv(tmplist,scr)
}

//** fillv(NAME,v1[,NAME1,v2 ...])
// fill from vectors v1,v2,..., places in ind -- use with selcp=0
proc fillv () { local i,fl
  if (numarg()==0) { 
    printf("fillv(NAME,vec1[,NAME1,vec2 ...])\n\tfill selected vectors from vectors\n")
    printf("\tuse after select() with selcp==0\n")
    return 
  }
  tmplist.remove_all vlist.remove_all
  for (i=1;i<=numarg();i+=2) {
    if (argtype(i)==2) { 
      if ((fl=fi($si))==-1) return
    } else fl=$i
    tmplist.append(v[fl])
  }
  for (i=2;i<=numarg();i+=2) vlist.append($oi)
  ind.resize(v.size)
  ind.sindx(tmplist,vlist)
}

//** pr() print out vectors
// eg pr("COLA","COLB",3,7)
func pr () { local ii,i,min,max,na,flag,jj,e
  if (eqobj(cob,out) && verbose) printf("Selected: ") 
  if (m==0) {print "EMPTY" return 0}
  flag=min=0 max=cob.v.size-1 na=numarg()
  if (na>=2) { 
    if (argtype(na-1)==0) { 
      i=na na-=2
      max=$i i-=1 min=$i 
      flag=1 // took care of numbers
    }} 
  if (!flag && na>=1) { 
    if (argtype(na)==0) { 
      i=na na-=1
      if ($i>=0) max=$i else min=max+$i // allow printing the end
    }} 
  flag=0 // reuse flag -- means printing only certain cols
  if (na>=1) if (argtype(1)==2) flag=1 // column names
  printf(" ")
  if (max>size(1)){ max=size(1)-1
    printf("NQS:pr WARNING: %d rows requested but %s size=%d\n",max,this,size(1)) }
  if (min>size(1)){printf("NQS:pr ERROR: %s size=%d < min %d\n",this,size(1),min) return 0}
  if (flag) {
    scr[1].resize(0)
    for i=1,na if ((ii=fi($si))!=-1) {
      scr[1].append(ii) 
      if (ii<0) printf("is[-ii].s\t") else printf("%s(%d)\t",s[ii].s,ii)
    } else return -1
    print ""
    for jj=min,max {
      for i=0,scr[1].size-1 { ii=scr[1].x[i]
        if (ii==-2) { printf("%g\t",cob.scr.x[jj]) 
        } else {
          prtval((e=getval(ii,cob.v[ii].x[jj])),"\t")         
          if (e==ERR) return ERR
        }
      }
      print ""
    }
  } else {
    for ii=0,m-1 printf("%4.4s  ",s[ii].s)
    print ""
    for jj=min,max {
      for ii=0,m-1 { prtval(e=getval(ii,cob.v[ii].x[jj]),"  ")
                     if (e==ERR) return  ERR}
      print ""
    }
  }
  return max-min+1
}

//** prn() print out single index from vectors
proc prn () { local jj,ii,ix,max,e
  ix=$1
  if (eqobj(cob,out) && verbose) printf("Selected: ") 
  if (numarg()==2) max=$2 else max=ix
  for jj=ix,max {
    if (jj<0 || jj>=cob.v[0].size) { 
      printf("prn: Index out of range (%d)\n",cob.v[0].size) return }
    for ii=0,m-1 {
      printf("%s:",s[ii].s)
      prtval(e=getval(ii,cob.v[ii].x[jj])," ")
      if (e==ERR) return
    }
    print ""
  }
}

//** zvec() -- clear -- resize all the vectors to 0
proc clear () { zvec() }
proc zvec () { local ii
  cob=this
  // fcds.remove_all fcds.append(new String("`EMPTY'"))
  for ii=0,m-1 { 
    if (numarg()==1) { v[ii].resize($1) v[ii].fill(-1) }// resize the buffer if desirable
    v[ii].resize(0) 
  }
}

//** pad() -- bring all vectors up to same length (of v[0])
func pad () { local sz
  cob=this
  if (numarg()==1) sz=$1 else sz=v.size
  for ii=0,m-1 { 
    if (v[ii].size>sz) printf("NQS.pad WARNING: neg padding %d\n",ii)
    v[ii].resize(sz)
  }
  return sz
}

//** size() -- return num of vectors and size of each vector
func size () { local ii,sz
  if (numarg()==1) {
    sz=cob.v.size // with 1 arg don't print anything
    for ii=1,m-1 if (cob.v[ii].size!=sz) sz=-1 // generate err
    return sz
  }
  if (m==0) { print "0 x 0" return 0 } // empty
  if (eqobj(cob,out) && verbose) printf("Selected: ") 
  printf("%d x %d",m,cob.v.size)
  for ii=1,m-1 printf(",%d",cob.v[ii].size)
  print ""
  return cob.v.size
}

//** resize(#cols[,#rows]) -- augment or dec the number of vectors
// resize("LABEL1","LABEL2",...)
func resize () { local oldsz,newsz,i,ii,jj,vsz,na,appfl
  na=numarg()
  vsz=-1
  if (na==1) { 
    newsz=$1 appfl=0 
  } else if (na==2 && argtype(1)==0 && argtype(2)==0) { 
    newsz=$1 appfl=0 
    vsz=$2
  }  else {
    if (int(na/2)!=na/2) { print "NQS Resize ERR: require even # of args"  return -1}
    newsz=m+numarg()/2
    appfl=1
  }
  oldsz=m
  if (m==newsz) { printf("clearing %s\n",this)
  } else if (newsz>m) {
    tmplist.remove_all
    for ii=0,m-1 { tmplist.append(v[ii]) tmplist.append(s[ii]) }
    objref v[newsz],s[newsz]
    jj=-1
    for ii=0,m-1 { v[ii]=tmplist.object(jj+=1) s[ii]=tmplist.object(jj+=1) }
    for ii=m,newsz-1 { v[ii]=new Vector() s[ii]=new String() }
    m=newsz
    tmplist.remove_all
  } else {
    for (ii=m-1;ii>=newsz;ii-=1) { v[ii]=nil s[ii]=nil }
    m=newsz
  }
  if (isassigned(out)) { 
    out.resize(m) // avoid recursion
    for ii=0,m-1 { out.v[ii].resize(0) out.s[ii].s="" }
  }
  x.resize(m) x.fill(0) fcd.resize(m)
  if (vsz>=1) for ii=0,m-1 v[ii].resize(vsz)
  if (appfl) { // append
    for (ii=1;ii<=na;ii+=2) { i=ii
      if (argtype(i)!=2) { printf("NQS RESIZE ERR: arg %d should be str\n",i) return -1}
      s[oldsz+(ii-1)/2].s=$si  i+=1
      if (argtype(i)==0) { 
        if ($i>0) v[oldsz+(ii-1)/2].resize($i)
      } else if (argtype(i)==1) { 
        v[oldsz+(ii-1)/2].copy($oi)
      } else {  printf("NQS RESIZE ERR2: arg %d should be num or obj\n",i) return -1}
    }
  }
  cob=this
  return m
}

//** sv(FNAME[,APPEND]) save the NQS
// to sv selected -- NQS.select(...) NQS.cp(NQS.out,1) NQS.sv()
proc sv () { local i,j
  file=$s1
  if (numarg()==2) { tmpfile.aopen(file)
  } else {
    if (tmpfile.ropen(file)) {
      if (batch_flag) {
        printf("NQS sv WARNING overwriting %s\n",file)
      } else if (!boolean_dialog("File exists","Overwrite","Cancel")) { 
          print "Cancelled" return
      }
    }
    if (! tmpfile.wopen(file)) { printf("ERR: can't open file\n") return }
  }
  savenums(m,fcds.count,(cnt=fcd.count(-1)))
  wrvstr(file) wrvstr(comment)
  for i=0,m-1 wrvstr(s[i].s)
  fcd.vwrite(tmpfile)
  for i=0,fcds.count-1 wrvstr(fcds.object(i).s)
  if (cnt>0) for i=0,fcd.size-1 if (fcd.x[i]==-1) { 
    savenums(fcdl.object(i).count)
    for j=0,fcdl.object(i).count-1 wrvstr(fcdl.object(i).object(j).s)
  }
  for i=0,m-1 {
    if (v[i].ismono(0)) { // only 1 value
      savenums(-1e9,v[i].size,v[i].x[0])
    } else v[i].vwrite(tmpfile)
  }
  x.vwrite(tmpfile)
  tmpfile.close
}

//** rd(FNAME[,CONTINUOUS]) read format saved by sv()
func rd () {
  if (numarg()==1) if (!tmpfile.ropen($s1)) { printf("ERR: can't open file\n") return 0 }
  resize(0)
  cnt=fc=0
  readnums(&ii,&fc,&cnt) // backward compatible -- if only 2 vals then cnt=0
  if (ii!=m) {
    m=ii
    objref v[m],s[m]
    for ii=0,m-1 { v[ii]=new Vector() s[ii]=new String() }
    x = new Vector(m) scr=x.c ind=x.c
  }
  rdvstr(file) rdvstr(comment)
  if (sfunc.len(file)==0) file=$s1
  for i=0,m-1 rdvstr(s[i].s)
  fcd.vread(tmpfile)
  fcds.remove_all
  if (isassigned(fcdl)) fcdl.remove_all
  for i=0,fc-1 { Xo=new String() fcds.append(Xo) rdvstr(Xo.s) }
  if (cnt>0) for i=0,fcd.size-1 if (fcd.x[i]==-1) { 
    readnums(&cnt)
    Yo=new List()
    for j=0,cnt-1 {Xo=new String() Yo.append(Xo) rdvstr(Xo.s)}
    useslist(i,Yo)
  }
  for i=0,m-1 { 
    v[i].vread(tmpfile)
    if (v[i].x[0]==-1e9) { v[i].resize(v[i].x[1]) v[i].fill(v[i].x[2]) }
  }
  x.vread(tmpfile)
  out.cp(this,0) // leave vectors empty
  return 1
}

//** func rdcols()
func rdcols () { local ii,cols,li,errflag,num
  errflag=0
  if (! tmpfile.ropen($s1)) { printf("\trdcols ERR0: can't open file \"%s\"\n",$s1) return 0}
  if (tmpfile.scanstr(sstr)==-1) {printf("\trdcols ERR1: file \"%s\"\n",$s1) return 0}
  if (isnum(sstr)){printf("\trdcols ERR2: no labels in file \"%s\"\n",$s1) return 0}
  cols=0
  while (! isnum(sstr)) { cols+=1 tmpfile.scanstr(sstr) }
  // 'grep -cve' takes 10x longer than 'wc -l' but skips blank lines
  sprint(sstr,"grep -cvP '^ *$' %s > /tmp/xtmp",$s1) system(sstr)
  tmpfile.ropen("/tmp/xtmp") li=tmpfile.scanvar()-1
  printf("%d cols; %d lines of data in %s.\n",cols,li,$s1)
  tmpfile.ropen($s1)
  tmpfile.gets(sstr) // remove first line
  num=scr.scanf(tmpfile,li*cols)
  if (num!=li*cols) { // err not reached since scanf dumps out
    printf("WARNING: expected %d vals; found %d\n",li*cols,num) errflag=3 }
  if (tmpfile.scanstr(sstr)>-1) { 
    printf("WARNING: %s found after reading in %d vals\n",sstr,li*cols) errflag=4 }
  resize(cols)
  tmpfile.seek(0)
  for ii=0,cols-1 { 
    tmpfile.scanstr(s[ii].s)
    v[ii].resize(li)
    v[ii].copy(scr,0,ii,li*cols-1,1,cols) // v[ii].mcol(scr,ii,cols)
  }  
  if (errflag) { printf("rdcols ERR%d\n",errflag) return 0 }
  return cols
}

//** func svcols(filename)
// currently only saves numeric columns
func svcols () { local ii,jj,cols,li,errflag,num
  errflag=0
  if (! tmpfile.wopen($s1)) { printf("\trdcols ERR0: can't open file \"%s\"\n",$s1) return 0}
  sstr2="\t"  // delimiter
  for ii=0,m-1 tmpfile.printf("%s%s",s[ii].s,sstr2)
  tmpfile.printf("\n")
  for ii=0,size(1)-1 {
    for jj=0,m-1 {
      getval(jj,v[jj].x[ii]) 
      tmpfile.printf("%g%s",nval,sstr2)
    }
    tmpfile.printf("\n")
  }
  tmpfile.close
  return ii
}

//** join(nqs1,nqs2,"FIELD")
// nqs1 will typically be preselected -- nqs2 will be appended to this
// index field can show up more than once in nqs1 but should only be once in nqs2
func join () { local vn1,vn2,ii,jj,kk,typ
  if ((vn1=$o1.fi($s3))==-1 || (vn2=$o2.fi($s3))==-1) { 
    printf("NQS::join() %s not found in both %s %s\n",$s3,$o1,$o2) return -1 }
  cp($o1) resize($o1.m+$o2.m) pad()
  fcd.resize($o1.m) fcd.append($o2.fcd)
  for jj=0,$o2.m-1 { kk=$o1.m+jj s[kk].s=$o2.s[jj].s }
  for ii=0,v.size-1 {
    typ=get(vn1,ii,nval,oval,sval)
    if (typ==0) {        num=$o2.select($s3,EQU,nval)
    } else if (typ==2) { num=$o2.select($s3,SEQ,sval)
    } else {printf("NQS::join ERRa: OBJ field not implemented: %s %s\n",$s3,oval) return -1}
    if (num==0) { printf("NQS::join ERRb: %s not found in %s\n",$s3,$o2) return -1 }
    if (num>1)  printf("NQS::join WARN: #%s >1 in %s\n",$s3,$o2)
    for jj=0,$o2.m-1 { kk=$o1.m+jj
      if ((fcd.x[jj])==0) { 
        v[kk].x[ii]=$o2.out.v[jj].x[0] // first entry from select
      } else if ((fcd.x[jj])==2) { 
        sval=$o2.fcds.object($o2.out.v[jj].x[0]).s
        v[kk].x[ii]=newval(2,kk)
      } 
    }
  }
  return m
}

//** jn(nqs2,"FIELD","COL1","COL2",...) // just append a new column onto this nqs
// index field can show up more than once in nqs1 but should only be once in nqs2
func jn () { local i,vn,vn1,vn2,ii,jj,kk,mm,typ,na,oldm,a
  na=numarg()
  if (na<=2) { printf("jn(nqs2,FIELD,COL1,COL2,...) append these cols from nqs2\n") return 0 }
  if ((vn1=fi($s2))==-1 || (vn2=$o1.fi($s2))==-1) { 
    printf("NQS::jn() %s not found in both %s\n",$s2,$o1) return -1 }
  oldm=m
  resize(m+na-2) pad()
  a=allocvecs(1)
  for i=3,na { kk=oldm+i-3
    s[kk].s=$si 
    if ((vn=$o1.fi($si))==-1){printf("NQS::jn() %s not found in %s\n",$si,$o1) return -1 }
    mso[a].append(vn)
    fcd.x[kk]=$o1.fcd.x[vn]
  }
  for ii=0,v.size-1 {
    typ=get(vn1,ii,nval,oval,sval)
    if (typ==0) {        num=$o1.select($s2,EQU,nval)
    } else if (typ==2) { num=$o1.select($s2,SEQ,sval)
    } else {printf("NQS::jn ERRa: OBJ field not implemented: %s %s\n",$s2,oval) return -1}
    if (num==0) {printf("NQS::jn ERRb: %s not found in %s\n",$s2,$o1) return -1 }
    if (num>1)  printf("NQS::jn WARN: #%s >1 in %s\n",$s2,$o1)
    for jj=0,mso[a].size-1 { kk=oldm+jj mm=mso[a].x[jj]
      if (($o1.fcd.x[mm])==0) { 
        v[kk].x[ii]=$o1.out.v[mm].x[0] // first entry from select
      } else if (($o1.fcd.x[mm])==2) { 
        sval=$o1.fcds.object($o1.out.v[mm].x[0]).s
        v[kk].x[ii]=newval(2,kk)
      } else { printf("NQS::jn ERRc fcd.x[%d]==%d\n",mm,fcd.x[mm]) return -1 }
    }
  }
  dealloc(a)
  return m
}

//** cp(NQS[,VEC_COPY]) copy 1 NQS to another
// default: VEC_COPY==1; side effect of NO_VEC_COPY is no fcd,fcds creation
proc cp () { local ii,csz,veccp
  csz=$o1.m
  if (numarg()==2) veccp=$2 else veccp=1
  if (m!=csz) {
    m=csz
    objref v[m],s[m]
    for ii=0,m-1 { v[ii]=new Vector() s[ii]=new String() }
    x = new Vector(m) scr=x.c ind=x.c
  }
  objl.remove_all
  for ii=0,$o1.objl.count-1 { objl.append($o1.objl.object(ii)) }
  file=$o1.file comment=$o1.comment
  for ii=0,m-1 { 
    s[ii].s=$o1.s[ii].s 
    if (veccp) v[ii].copy($o1.v[ii]) // 2nd arg to not copy vectors
  }
  if (veccp==1) {  // full copy
    fcd.copy($o1.fcd) 
    for ii=0,$o1.fcds.count-1 fcds.append($o1.fcds.object(ii))
    if (isobj($o1.fcdl,"List")) { fcdl=new List()
      for ii=0,$o1.fcdl.count-1 fcdl.append($o1.fcdl.object(ii)) 
    }
  } else if (! isassigned(fcd)) { // use pointers for .out
    fcd=$o1.fcd fcds=$o1.fcds fcdl=$o1.fcdl tmplist=$o1.tmplist
  } 
  x.copy($o1.x) x.resize(m)
  scr.copy($o1.scr) ind.copy($o1.ind)
  if (isassigned(out)) { 
    out.resize(m)
    for ii=0,m-1 { out.v[ii].resize(0) out.s[ii].s="" }
  }
}

//** eq(NQS) -- just check the vecs
func eq () { local ii,jj
  if ($o1.m!=m) { printf("# of cols differ %d vs %d\n",m,$o1.m) return 0 }
  for ii=0,m-1 if (strcmp($o1.s[ii].s,s[ii].s)!=0) { 
    printf("%d col names differ: %s vs %s",ii,s[ii].s,$o1.s[ii].s) return 0 }
  for ii=0,m-1 if ($o1.v[ii].size != v[ii].size) { 
    printf("%d col lengths differ: %d vs %d",ii,v[ii].size,$o1.v[ii].size) return 0 }
  for ii=0,m-1 if (! $o1.v[ii].eq(v[ii])) { 
    printf("%s cols differ at ",s[ii].s)
    for jj=0,v[ii].size-1 if ($o1.v[ii].x[jj] != v[ii].x[jj]) {
      printf("element %d: %g vs %g",jj,v[ii].x[jj],$o1.v[ii].x[jj])
      return 0
    }
  }
  if (! fcdseq($o1)) return 0
  return 1
}

//** fcdseq() -- check that string lists are identical in two NQSs -- this is
// sufficient but not nec for comparing string columns for JOIN
// in order to use JOIN must share same fcds by setting up with strdec(NQS,...)
// (could break out separate lists for each str column -- tried in nqs.hoc220)
func fcdseq () { local ii,jj,cnt
  cnt=fcds.count
  if (eqobj(fcds,$o1.fcds)) {
    printf("%s %s share string list fcds\n",this,$o1)
    return 1
  }
  if (cnt!=$o1.fcds.count) {
    printf("DIFFERING (1) string lists (fcds) %d %d\n",fcds.count,$o1.fcds.count)
    return 0
  }
  for ii=0,cnt-1 if (!strcmp(fcds.object(ii).s,$o1.fcds.object(ii).s)==0) {
    printf("DIFFERING (2) string lists (fcds) %d:%s vs %s",ii,fcds.object(ii).s,$o1.fcds.object(ii).s)
    return 0
  }
  if (! fcd.eq($o1.fcd)) {
    printf("DIFFERING (3) col keys (fcd) ") vlk(fcd) vlk($o1.fcd)
    return 0
  }
  if (! isassigned(fcdl) && isassigned($o1.fcdl)) {
      printf("DIFFERING (4) uselists() string lists: absent in %s\n",this)
      return 0
  }
  if (isassigned(fcdl)) {
    if (! isassigned($o1.fcdl)) {
      printf("DIFFERING (5) uselists() string lists absent in %s\n",$o1)
      return 0
    }
    if (fcdl.count!=$o1.fcdl.count) {
      printf("DIFFERING (6) uselists() list list counts %d vs %d",fcdl.count,$o1.fcdl.count)
      return 0
    }
    for ii=0,fcdl.count-1 if (fcd.x[ii]==-1) {
      if (!isobj(fcdl.object(ii),"List") || !isobj($o1.fcdl.object(ii),"List")) {
        printf("DIFFERING (7) uselists() string lists (fcdl.obj) %d:%s vs %s",ii,\
                   fcdl.object(ii),$o1.fcdl.object(ii))
        return 0
      }
      if (fcdl.object(ii).count != $o1.fcdl.object(ii).count) {
        printf("DIFFERING (8) uselists() string lists counts (fcdl.obj) %d:%d vs %d",ii,\
                   fcdl.object(ii).count,$o1.fcdl.object(ii).count)
        return 0
      }
      for jj=0,fcdl.object(ii).count-1 {
        if (!strcmp(fcdl.object(ii).object(jj).s,$o1.fcdl.object(ii).object(jj).s)==0) {
          printf("DIFFERING (9) uselists() string lists (fcdl.obj) %d,%d:%s vs %s",ii,jj,\
                 fcdl.object(ii).object(jj).s,$o1.fcdl.object(ii).object(jj).s)
          return 0
        }
      }
    }
  }
  return 1
}

//** strdec() -- declare these columns to be strings
func strdec () { local i,min
  min=1
  if (eqobj(cob,out)) {
    printf("str() ERR: string fields can only be declared at top level\n") return 0}
  if (numarg()==0) { 
    printf("str(NAME[,NAME1 ...])\n\tdeclare these field to be string fields\n") return 0}
  out.fcd=fcd
  if (argtype(1)==1) {
    if (fcds.count>0) if (! fcdseq($o1)) { 
      printf("Pre-existing string lists differ; unable to join %s %s\n",this,$o1)
      return 0
    }
    fcds=$o1.fcds // share string list to allow JOIN on a string field
    min=2 
  }
  for i=min,numarg() { fl=fi($si)
    if (fl>-1) {
      fcd.x[fl]=2
      sval="`EMPTY'"
      newval(2,fl)   // don't want to put on more than one
    }
  }
  return 1
}

//** mo([flag][,STAT1,...]) -- create global objectvars that point to the vectors
// first time use flag=1 to create new global objrefs, else just shift them
// flag=1 reassign objl but don't care if they already exist
// flag=2 don't print out the assignments
// flag=3 reassign objl; make sure they're unique
// flag=4 clear the vectors
// should we also create a set of global scalars to assign to in an iterator?
proc mo () { local ii,flag,i,hf
  if (numarg()>=1) flag=$1 else flag=0 // flag:create objrefs
  if (flag==1 || flag==3) {
    if (objl.count>0) {
      if (flag==3) if (batch_flag) {
        printf("NQS mo(3) WARNING: Renamed object pointers.\n")
      } else if (! boolean_dialog("New name object pointers?","YES","NO")) return
      if (flag==1) if (batch_flag) {
        printf("NQS mo(1) WARNING: Rassigned object pointers.\n")
      } else if (! boolean_dialog("Reassign object pointers?","YES","NO")) return
    }
    objl.remove_all
    for ii=0,m-1 if (sfunc.len(s[ii].s)>0) {
      sstr=s[ii].s
      repl_mstr(sstr,"[^A-za-z0-9]","",execstr)
      sprint(sstr,"%sv",sstr)
      if (flag==3) { // make sure it's unique
        hf=0
        while (name_declared(sstr)) { hf=1
          printf("%s exists ... ",sstr)
          sprint(sstr,"%sv",sstr) 
        } 
        if (hf) printf(" -> %s\n",sstr)
      } else if (name_declared(sstr)) printf("%s reassigned: ",sstr)
      printf("%s -> v[%d] (%s)\n",sstr,ii,s[ii].s)
      sprint(execstr,"objref %s",sstr) execute(execstr)
      sprint(execstr,"%s=%s",sstr,v[ii]) execute(execstr)
      objl.append(new String(sstr))
    }
    sprint(execstr,"objref indv") execute(execstr)
    sprint(execstr,"indv=%s",ind) execute(execstr)
  } else {
    if (objl.count==0) { 
      printf("Must create vecs with mo(1)\n") 
    } else if (objl.count>m) { 
      printf("STAT:mo ERR: wrong objref count in objl: %d vs %d\n",objl.count,m)
      return
    } else {
      if (objl.count<m) { 
        printf("STAT:mo WARNING: unreferenced vecs for %s: refs %d<m %d\n",this,objl.count,m) }
      for ii=0,objl.count-1 {
        Xo=objl.object(ii)
        if (flag==0) printf("%s -> %s.v[%d] (%s)\n",Xo.s,this,ii,s[ii].s)
        if (flag==4) {sprint(execstr,"%s=nil",Xo.s) 
        } else        sprint(execstr,"%s=%s",Xo.s,v[ii]) 
        execute(execstr)
      }
    }
    sprint(execstr,"objref indv") execute(execstr)
    if (flag!=4) { sprint(execstr,"indv=%s",ind) execute(execstr) }
  }
  if (numarg()>1) for i=2,numarg() { // propagate the objl to other STATs
    $oi.objl.remove_all
    for ii=0,objl.count-1 $oi.objl.append(objl.object(ii))
  }
}

//* endtemplate
endtemplate NQS

//* ancillary routines
//** prl2nqs(NQS[,min,max,nointerp]) -- transfer printlist to NQS
proc rename () {}
interp=1  // default
// eg proc rename () { sprint($s1,"P%d",objnum($s1)) }
proc prl2nqs () { local tstep
  if (numarg()>=2) min=$2 else min=0
  if (numarg()>=3) max=$3 else max=printlist.count-1
  if (numarg()>=4) interp=$4 // no interp when looking at spk times
  if (interp) $o1.resize(max-min+2) else $o1.resize(2*(max-min+1))
  if (interp) {
    tstep=0.1 // 0.1 ms step size for interpolation
    $o1.s[0].s="time"
    $o1.v[0].indgen(0,printlist.object(0).tvec.max,tstep)
    for ii=min,max {
      XO=printlist.object(ii)
      $o1.s[ii+1-min].s = XO.var
      rename($o1.s[ii+1-min].s)
      $o1.v[ii+1-min].resize($o1.v.size)
      $o1.v[ii+1-min].interpolate($o1.v[0],XO.tvec,XO.vec)
    }
  } else {
    for ii=min,max {
      XO=printlist.object(ii)
      $o1.s[2*(ii-min)+1].s = XO.var
      rename($o1.s[2*(ii-min)+1].s)
      sprint($o1.s[2*(ii-min)].s,"%s-time",$o1.s[2*(ii-min)+1].s)
      $o1.v[2*(ii-min)].copy(XO.tvec)
      $o1.s[2*(ii-min)+1].s = XO.var
      rename($o1.s[2*(ii-min)+1].s)
      $o1.v[2*(ii-min)+1].copy(XO.vec)
    }
  }
}

//** pvp2nqs(NQS) -- transfer grvec data file to NQS
proc pvp2nqs () { local tstep,tv1,v1
  if (! read_vfile(1,$s1)) { printf("Can't open %s\n",$s1) return }
  if (numarg()>=3) min=$3 else min=0
  if (numarg()>=4) max=$4 else max=panobj.llist.count-1
  if (numarg()>=5) interp=$5 // no interp when looking at spk times
  if (interp) $o2.resize(max-min+2) else $o2.resize(2*(max-min+1))
  if (interp) {
    tv1=v1=allocvecs(2)  v1+=1
    tstep=0.1 // 0.1 ms step size for interpolation
    $o2.s[0].s="time"
    for ii=min,max {
      XO=panobj.llist.object(ii)
      $o2.s[ii+1-min].s = XO.name
      rename($o2.s[ii-min+1].s)
      tmpfile.seek(XO.loc) mso[tv1].vread(tmpfile) mso[v1].vread(tmpfile) // or use rv_readvec
      if (ii-min==0) $o2.v[0].indgen(0,mso[tv1].max,tstep)
      $o2.v[ii+1-min].resize($o2.v.size)
      $o2.v[ii+1-min].interpolate($o2.v[0],mso[tv1],mso[v1])
    }
    dealloc(tv1)
  } else for ii=min,max {
    XO=panobj.llist.object(ii)
    $o2.s[2*(ii-min)].s = XO.name
    rename($o2.s[2*(ii-min)].s)
    sprint($o2.s[2*(ii-min)].s,"%s-time",$o2.s[2*(ii-min)].s)
    $o2.s[2*(ii-min)+1].s = XO.name
    rename($o2.s[2*(ii-min)+1].s)
    tmpfile.seek(XO.loc)
    $o2.v[2*(ii-min)].vread(tmpfile)
    $o2.v[2*(ii-min)+1].vread(tmpfile)
  }
}

//** veclist2nqs(nqs[,STR1,STR2,...])
proc veclist2nqs () { local flag,i
  if (numarg()==0) {printf("veclist2nqs(nqs[,STR1,STR2,...])\n") return}
  $o1.resize(veclist.count)
  if (numarg()==1+$o1.m) flag=1 else flag=0
  for ltr(XO,veclist) {
    $o1.v[i1].copy(XO)
    if (flag) {i=i1+2 $o1.s[i1].s=$si} else {sprint(tstr,"v%d",i1) $o1.s[i1].s=tstr}
  }
  $o1.out.cp($o1,0)
}

//* code routines
//** mkcode() creates a code of 16 cols in 4 fields, sized 4,4,4,4
// each field can store a number from 0-9999
double cdsep[5]
cdsep[0] = 1e16
cdsep[1] = 1e12
cdsep[2] = 1e8 
cdsep[3] = 1e4
cdsep[4] = 1e0
func mkcode () { 
  // divide into 4 fields of size 3,3,3,3
  return $1*cdsep[1] + $2*cdsep[2] + $3*cdsep[3] + $4*cdsep[4]
}

//** cd(i,code) returns field i (1-4) from code
// $1 field number (from 1), $2 code
func cd () { return int($2%cdsep[$1-1]/cdsep[$1]) }

//** setcd(i,code,new) replaces field i (1-4) from code with new
// $1 field number (1 offset), $2 code, $3 new value
func setcd () { local old
  old = cdsep[$1]*int(($2%cdsep[$1-1])/cdsep[$1])
  return $2 - old + $3*cdsep[$1]
}

//** prcode(code) prints a code in readable form
proc prcode () { local i
  for (i=1;i<4;i=i+1) printf("%d,",cd(i,$1))
  printf("%d\n",cd(4,$1))
}

//** sopset() returns symbolic arg associated with a string
proc sopset() { local i
  for i=1,19 { sops[i-1]=$i } // AUGMENT TO ADD NEW OPSYM
}
sopset(ALL,NEG,POS,CHK,NOZ,GTH,GTE,LTH,LTE,EQU,EQV,EQW,NEQ,SEQ,RXP,IBE,EBI,IBI,EBE) // ADD NEW OPSYM NAME

//** whvarg
func whvarg () { local ret
  ret=-1
  // ADD NEW OPSYM STRING
          // ALL  NEG   POS  CHK  NOZ   GTH  GTE  LTH LTE EQU   EQV   EQW   NEQ  SEQ  RXP  IBE  EBI  IBI   EBE
  for scase("ALL","<0",">0","CHK","!=0",">",">=","<","<=","==","EQV","EQW","!=","=~","~~","[)","(]","[]","()") {
    if (strcmp($s1,temp_string_)==0) {ret=i1 break}
  }
  if (ret==-1) return ret else return sops[ret]
}

//** whkey(KEY,STR) 
// place name of KEY from vecst.mod in temp_string_
proc whkey () { local key
  for scase("ALL","NEG","POS","CHK","NOZ","GTH","GTE","LTH","LTE","EQU","EQV","EQW","NEQ","SEQ","RXP","IBE","EBI","IBI","EBE") { // ADD NEW OPSYM NAME
    sprint(tstr,"x=%s",temp_string_) execute(tstr)
    if (x==$1) { $s2=temp_string_ break }
  }
}

//** varstr(tstr) -- make a variable out of string by removing nonalphanumeric characters
func varstr () { local a,z,A,Z,a0,a9,a_,len,ii,sflag
  a=97 z=122 A=65 Z=90 a0=48 a9=57 a_=95 // ascii codes
  if (numarg()==2) sflag=1 else sflag=0
  len = sfunc.len($s1)
  for ({x=0 ii=0};ii<len && !((x>=a&&x<=z)||(x>=A&&x<=Z));ii+=1) { // allowed first char
    sscanf($s1,"%c%*s",&x)
    sfunc.right($s1,1)
  }
  if (ii==len) { printf("varstr() ERR: no useable characters") return 0}
  sprint($s1,"%c%s",x,$s1)
  for (;ii<=len;ii+=1) {
    sscanf($s1,"%c%*s",&x)
    sfunc.right($s1,1)
    if ((x>=a&&x<=z)||(x>=A&&x<=Z)||(x>=a0&&x<=a9)||(x==a_)) { // allowed chars
      sprint($s1,"%s%c",$s1,x)
    }
  }
  if (sflag) { 
    sprint($s1,"strdef %s",$s1) 
    execute($s1) 
    sfunc.right($s1,7) // strip leading "strdef"
  } else {
    sprint($s1,"%s=0",$s1) 
    execute($s1) 
    sfunc.left($s1,sfunc.len($s1)-2) // strip the =0
  }
  return 1
}

strdef h1
h1="Select operators: \nALL <0  >0  CHK !=0  >  >=  <   <=  ==  EQV EQW !=  =~  ~~  [)  (]  []  ()\nALL NEG POS CHK NOZ GTH GTE LTH LTE EQU EQV EQW NEQ SEQ RXP IBE EBI IBI EBE\nmost are obvious; those that are not\nEQV: value equal to same row value another column (takes string arg)\nEQW: value found in other vector (takes vector or NQS arg)\nSEQ: string equal (takes string)\nRXP: regular expression comparison (takes string)\nIBE,EBI...: I=inclusive, E=exclusive for ranges\n"

proc nqshelp () {
  if (numarg()==0) {
  } else {
    if (strcmp($s1,"select")==0) {
      // ed=new TextEditor(h1,9,160) ed.map
      printf("%s",h1)
    }
  }
}

//* nqsdel(NQS_objref) fully delete an nqs
proc nqsdel () {
  $o1.out.cob=nil  $o1.out=nil
  $o1.cob=nil $o1=nil
}

Loading data, please wait...