Skip to content
Snippets Groups Projects
Commit 6efceef6 authored by Aflitos, Saulo Alves's avatar Aflitos, Saulo Alves
Browse files

first

parents
No related branches found
No related tags found
No related merge requests found
Expand all Show whitespace changes
Inline Side-by-side
Showing
with 639 additions and 0 deletions
.gitignore 0 → 100644
+ 8
− 0
View file @ 6efceef6
*.tar.gz
*.dblite
*.fai
*.pyc
*.dat
*.dot
*.png
*.html
FRC 0 → 120000
+ 2
− 0
View file @ 6efceef6
/home/assembly/nobackup/metrics/FRC_align/src/FRC
\ No newline at end of file
SConstruct 0 → 120000
+ 2
− 0
View file @ 6efceef6
run2.py
\ No newline at end of file
SConstruct.graph.py 0 → 100755
+ 390
− 0
View file @ 6efceef6
#!/usr/bin/python
import sys,os
import datetime
ignore_list = ['/bin/echo']
#['/mnt/nexenta/aflit001/nobackup/Data/', '']
replace = [ ['Data/', ''], ['out/', ''], ['data/', '']]
def parseStatus(status):
#01234567891
# E = exists
# R = exists in repository only
# b = implicit builder
# B = explicit builder
# S = side effect
#01234567891
# P = precious
# A = always build
# C = current
# N = no clean
# H = no cache
#01234567891
statusF = 0
if status[0] == "E": statusF += 1
if status[1] == "R": statusF += 2
if status[2] == "b": statusF += 4
if status[3] == "B": statusF += 4
if status[3] == "S": statusF += 8
if status[4] == "P": statusF += 16
if status[5] == "A": statusF += 32
if status[6] == "C": statusF += 64
if status[7] == "N": statusF += 128
if status[8] == "H": statusF += 256
return statusF
def clean_for_dot(expr):
expr = expr.strip()
for rep in replace:
#print "REPLACING",rep[0],'to',rep[1],'in',expr,
expr = expr.replace(rep[0],rep[1])
#print " BECOMING",expr
if expr[0] == '/': expr = os.path.basename( expr )
expr = expr.replace("/","\/").replace("-","\-").replace("_","\_")
if expr[0] == "[": expr = expr[1: ]
if expr[-1] == "]": expr = expr[:-1]
return expr
class node(object):
def __init__(self,path, status, depth):
self.path = path
self.status = status
self.depth = depth
self.children = []
self.statusF = parseStatus(self.status)
def add_child(self,child):
if not child in self.children: self.children.append(child)
def print_rec(self,depth):
print " "*depth,self.name
for c in self.children:
c.print_rec(depth+1)
def get_name(self):
return self.path
def get_label_short(self):
return self.path
def get_status(self):
return self.statusF
def get_depth(self):
return self.depth
def get_status_color(self):
color = 'white'
if (self.statusF % 2 ) != 0:
color = 'green'
return color
def is_fixed_tool(self):
return self.path in ignore_list
class scons():
def __init__(self):
self.stack = []
self.map = {}
def printOut(self):
o = open('SConstruct.tree.dot', 'w')
def wl(s): o.write("%s\n"%s)
wl("digraph G {")
##wl("""size="100x100" """)
##wl("""rankdir=TB """)
wl("raksep=equally")
#wl("nodesep=0.8")
wl("pagedir=BL")
wl("splines=true")
wl("concentrate=true")
wl("fontname=Helvetica")
wl("remincross=true")
wl("searchsize=100")
#wl("""rankdir=RL """)
#wl("ratio = 1")
#wl("ratio = compress")
wl("ratio = auto")
#Nodes:
for obj in self.map.values():
if not obj.is_fixed_tool():
#wl("""\"%s\" [label=\"%s\", fontsize=16, style=filled, fillcolor=%s]"""%(scons_obj.get_name(),scons_obj.get_label_short(),scons_obj.get_status_color()) )
#wl("""\"%s\" [label=\"%s\", nodesep=0.75, ranksep=0.75, shape=box, weight=1.2, fontsize=16, style=filled, fillcolor=%s]"""%(scons_obj.get_name(),scons_obj.get_label_short(),scons_obj.get_status_color()) )
wl("""\"%s\" [label=\"%s\", group=%s, rank=%s, shape=box, fontsize=24, style=filled, fillcolor=%s]"""%(obj.get_name(),obj.get_label_short(), obj.get_depth(), obj.get_depth(), obj.get_status_color()) )
#Connections:
for obj in self.map.values():
if not obj.is_fixed_tool() :
children = obj.children
if len(children) == 0:
continue
names = []
exts = []
for c in children:
if not c.is_fixed_tool():
name = c.get_name()
ext = os.path.splitext(name)
if len(ext) > 0:
ext = ext[1][1:]
names.append(name)
exts.append(ext)
nameext = zip(exts,names,children)
#sys.stderr.writelines("NAMEEXT 1"+str(nameext))
nameext.sort()
#sys.stderr.writelines("NAMEEXT 2"+str(nameext))
(sortedNames, sortedExts, sortedChildren) = zip( *nameext )
for c in sortedChildren:
if not c.is_fixed_tool():
wl("\"%s\" -> \"%s\""%(c.get_name(), obj.get_name() ) )
#wl("\"%s\" -> \"%s\""%(scons_obj.get_name(), c.get_name() ) )
wl("}")
def initial_filter(datalines):
return [d for d in datalines if d.strip() and d.strip()[0] in ["["] ]
class htmlTree():
def __init__(self):
self.lines = []
def checkRoot(self, str):
if str[0] == "[" and str[-1] == "]":
return False
else:
return True
def add(self, statusStr, pre, pos, nameClean, depth):
status = parseStatus(statusStr)
position = len(self.lines)
self.lines.append([status, pre, nameClean, depth, self.checkRoot(pos)])
def export(self):
lines = []
lines.append("<html>")
#www.perbang.dk/rgbgradient/ AD0000 E5E5FF 12 steps
lines.append("""
<style type="text/css">
p {
margin: 0px;
font-family: monospace;
white-space: nowrap;
display: inline;
}
p.parent { font-weight: 900 }
p.child { font-weight: 100 }
p.run { color: #00AA00 }
p.notrun { color: #FF0000 }
p { display: block }
</style>
<script src="http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
<script type="text/javascript">
function hide(name) {
$('.'+name).hide();
//alert('hiding '+name)
}
function show(name) {
$('.'+name).show();
//alert('showing '+name)
}
</script>
""")
#p.depth01 { background-color: #000000 }
#p.depth02 { background-color: #333333 }
#p.depth03 { background-color: #555555 }
#p.depth04 { background-color: #666666 }
#p.depth05 { background-color: #7A7A7A }
#p.depth06 { background-color: #8B8B8B }
#p.depth07 { background-color: #999999 }
#p.depth08 { background-color: #AAAAAA }
#p.depth09 { background-color: #BBBBBB }
#p.depth10 { background-color: #CCCCCC }
#p.depth11 { background-color: #DDDDDD }
#p.depth12 { background-color: #EEEEEE }
now = str(datetime.datetime.now())
print now
lines.append(" <body>")
lines.append("""
<table>
<tr>
<td>
Hide
</td>
<td>
<input type="button" onclick="hide('parent')" value="parent"/>
</td>
<td>
<input type="button" onclick="hide('child')" value="child"/>
</td>
<td>
<input type="button" onclick="hide('run')" value="run"/>
</td>
<td>
<input type="button" onclick="hide('notrun')" value="not run"/>
</td>
</tr>
<tr>
<td>
Show
</td>
<td>
<input type="button" onclick="show('parent')" value="parent"/>
</td>
<td>
<input type="button" onclick="show('child')" value="child"/>
</td>
<td>
<input type="button" onclick="show('run')" value="run"/>
</td>
<td>
<input type="button" onclick="show('notrun')" value="not run"/>
</td>
</tr>
</table>
""")
lines.append("LABEL:<br/>")
lines.append(' <p class="parent run" >'+'Finished Running - Original File'.replace(' ', '&nbsp;')+'</p>')
lines.append(' <p class="child run" >'+'Finished Running - Copy File'.replace(' ', '&nbsp;')+'</p>')
lines.append(' <p class="parent notrun" >'+'Not Run - Original File'.replace(' ', '&nbsp;')+'</p>')
lines.append(' <p class="child notrun" >'+'Not Run - Copy File'.replace(' ', '&nbsp;')+'</p>')
lines.append(' <p>'+now+'</p>')
#lines.append(' <p >Level:</p>')
#for p in range(1, 13):
# lines.append(' <p class="parent run depth%02d">&nbsp;%02d&nbsp;</p>' % (p ,p))
lines.append(' <br/><br/>')
for line in self.lines:
status = line[0]
pre = line[1]
name = line[2]
depth = line[3]
parent = line[4]
refname = name.replace('\/', '').replace('\\', '').replace('.', '')
name = name.replace('\\', '')
pre = pre.replace(' ', '&nbsp;')
linkStr = ''
parentHood = ''
runStatus = ''
if parent:
linkStr = '<a name="'+refname+'"></a>'
parentHood = 'parent'
else:
linkStr = '<a href="#'+refname+'">^</a>'
parentHood = 'child'
if (status % 2 ) != 0:
runStatus = 'run'
else:
runStatus = 'notrun'
lines.append(' <p class="%s %s depth%02d">' % (parentHood, runStatus, depth) + pre + linkStr + name + '</p>')
lines.append(" </body>")
lines.append("</html>")
f = open('SConstruct.tree.html', 'w')
f.writelines('\n'.join(lines))
f.close()
def main():
data = sys.stdin.readlines()
data = initial_filter(data)
HTML = htmlTree()
if len(sys.argv) > 1:
replace.append([sys.argv[1] + '/', ''])
replace.append([sys.argv[1] + '_', ''])
root = node("root",' '*9, 0)
scons_obj = scons()
scons_obj.stack.append( root )
def does_object_exist(name): return name in scons_obj.map.keys()
html = []
htmlKeys = {}
# Read the Data:
lc = 0
for l in data:
lc += 1
l = l.strip()
#print "LINE ",lc,"1",l
statusStr = l[1:10]
nameStr = l[11:]
nameClean = nameStr
depth = 1
for p in range(0, len(nameStr)):
if nameStr[p] not in ['\[', ' ', '-', '+', '|']:
#print "FOUND",nameStr[p]
pre = nameStr[0:p]
pos = nameStr[p:]
#pre = pre.replace("+",'').replace("-",'').replace("|",'') #Remove the "tree parts" from the text
depth = ( (p - 2)/2 ) + 1
nameClean = clean_for_dot(pos)
HTML.add(statusStr, pre, pos, nameClean, depth)
break
#print "LINE ",lc,"LINE",l,"STATUS STR",statusStr,"NAME STR",nameStr,"NAME CLEAN",nameClean,"DEPTH",depth
#if depth == 0: depth = 1
#print "LINE ",lc,"3 '",nodename,","
#if nodename[0] != "/": nodename = "./"+nodename
#print "LINE ",lc,"4 '",nodename,","
if not does_object_exist(nameClean):
#print "LINE ",lc,"5",nodename,"does not exists"
scons_obj.map[nameClean] = node(nameClean,statusStr,depth)
#print "LINE ",lc,"6",nodename,"node",node(nodename).get_name()
#depth = (len(l) - len(l.strip() ) +1)/2
#print "LINE ",lc,"7 DEPTH",depth
# If the depth is one higher than the depth of the stack,
# add as a child object.
if len(scons_obj.stack) == depth -1:
#print "LINE ",lc,"8 CHILD"
scons_obj.stack[-1].add_child( scons_obj.map[nameClean] )
scons_obj.stack.append( scons_obj.map[nameClean] )
# Otherwise pop the stack until we reach the parent... and add as a child
else:
#print "LINE ",lc,"9 NOT CHILD"
while( len(scons_obj.stack) > depth ) : scons_obj.stack.pop()
if len(scons_obj.stack) > 0:
scons_obj.stack[-1].add_child( scons_obj.map[nameClean] )
scons_obj.stack.append( scons_obj.map[nameClean] )
#print '-'*20
#root.print_rec(0)
HTML.export()
scons_obj.printOut()
main()
SConstruct.run 0 → 100755
+ 4
− 0
View file @ 6efceef6
#!/bin/bash
LOGFILE="SConstruct.run.log."`date +%Y%m%d_%H%M%S`".out"
#echo "LOGFILE $LOGFILE"
python scons/scons.py -n --stack-size=1024 --diskcheck=none $@ 2>&1 | tee $LOGFILE
SConstruct.runReal 0 → 100755
+ 4
− 0
View file @ 6efceef6
#!/bin/bash
LOGFILE="SConstruct.run.log."`date +%Y%m%d_%H%M%S`".out"
#echo "LOGFILE $LOGFILE"
python scons/scons.py --stack-size=1024 --diskcheck=none $@ 2>&1 | tee $LOGFILE
SConstruct.tree.sh 0 → 100755
+ 21
− 0
View file @ 6efceef6
#!/bin/bash
rm SConstruct.tree.dot 2>/dev/null
rm SConstruct.tree.png 2>/dev/null
rm SConstruct.tree.gif 2>/dev/null
rm SConstruct.tree.html 2>/dev/null
bash SConstruct.run -n --tree=prune,status $@ > SConstruct.tree.dat
#,derived
cat SConstruct.tree.dat | python SConstruct.graph.py $@
#dot -Tgif tree.dot > tree.gif
#dot -Tpng tree.dot > tree.png
# | tee tree.dot | dot -Tgif > tree.gif
#./run -n --tree=status $@ | tee tree.dat
#cat tree.dat | ./graph.py | neato -Tpng > neato_rendering.png
#cat tree.dat | ./graph.py | fdp -Tpng > fdp_rendering.png
#cat tree.dat | ./graph.py | dot -Tgif > tree.gif
cat SConstruct.tree.dot | /usr/bin/dot -x -Tpng > SConstruct.tree.png
#cat tree.dat | ./graph.py | circo -Tpng > circo_rendering.png
merger.py 0 → 100755
+ 183
− 0
View file @ 6efceef6
#!/usr/bin/python
import os, sys
import glob
from pprint import pprint as pp
if len(sys.argv) < 4:
print "no libs given"
print sys.argv[0], "<IN CSV GROUP> <IN CSV LIB> <OUT FILE>"
sys.exit(1)
CSV_GROUP = sys.argv[1]
CSV_LIB = sys.argv[2]
CSV_OUT = sys.argv[3]
if (not os.path.exists( CSV_GROUP )) or ((CSV_LIB is not None) and (not os.path.exists(CSV_LIB))):
if (not os.path.exists( CSV_GROUP )):
print "csv group file %s does not exists" % CSV_GROUP
if ((CSV_LIB is not None) and (not os.path.exists(CSV_LIB))):
print "csv lib file %s does not exists" % CSV_LIB
print sys.arv[0], "<IN CSV GROUP> <IN CSV LIB>"
sys.exit(1)
if os.path.exists( CSV_OUT ):
print "output file %s already exists" % CSV_OUT
sys.exit( 1 )
def main():
data = {}
groups = {}
with open(CSV_LIB, 'r') as fhd:
line_count = 0
for line in fhd:
line_count += 1
if line_count == 1:
continue
if len(line.strip()) == 0 or line[0] == '#':
continue
#0 1 2 3 4 5 6 7 8 9 10 11
#library_name, project_name, organism_name, type , paired, frag_size, frag_stddev, insert_size, insert_stddev, read_orientation, genomic_start, genomic_end
#frags_PE_170, arcanum , sa , fragment, 1 , 170 , 17 , , , inward , 0 , 0
cels = line.split(',')
print "parsing line '%s'" % line.strip()
gname = cels[0].strip()
fsize = cels[5].strip()
fstdd = cels[6].strip()
isize = cels[7].strip()
istdd = cels[8].strip()
if gname not in data:
groups[gname] = {}
gtype = ""
gsize = 0
gmin = 0
gmax = 0
gmax2 = 0
if len(fsize) > 0:
gtype = 'pe'
gsize = int(fsize)
gmin = int(fsize) - int(fstdd)
gmax = int(fsize) + int(fstdd)
gmax2 = int(fsize) + (3*int(fstdd))
elif len(isize) > 0:
gtype = 'mp'
#data[gname]['type'] = 'pe'
gsize = int(isize)
gmin = int(isize) - int(istdd)
gmax = int(isize) + int(istdd)
gmax2 = int(isize) + (3*int(istdd))
else:
print line
print "count not find size"
sys.exit(1)
groups[gname]['gtype'] = gtype
groups[gname]['gsize'] = gsize
groups[gname]['gmin' ] = gmin
groups[gname]['gmax' ] = gmax
groups[gname]['gmax2'] = gmax2
with open(CSV_GROUP, 'r') as fhd:
line_count = 0
for line in fhd:
line_count += 1
if line_count == 1:
continue
if len(line.strip()) == 0 or line[0] == '#':
continue
#0
#file_name , group_name, library_name
#120527_I247_FCC0U0NACXX_L7_TOMxitDGJDWAAPEI-93_*.fq.gz, MP_2000_1 , jumps_MP_2000
cels = line.split(',')
print "parsing line '%s'" % line.strip()
fname = cels[0].strip()
gname = cels[2].strip()
fnames = glob.glob(fname)
if len(fnames) == 0:
print "error globing '%s'" % fname.strip()
print " line '%s'" % line.strip()
sys.exit(1)
cpref = os.path.basename( os.path.commonprefix(fnames) )
cpref = cpref.rstrip("_.-")
print cpref
if len(fnames) % 2 != 0:
print "oops. failed to glob"
sys.exit(1)
elif len(fnames) == 2:
data[cpref] = { "file_names": [] }
for ffname in fnames:
print ffname
if not os.path.exists(ffname) or not os.path.isfile(ffname):
print "file %s does not exists" % ffname
sys.exit(1)
datal = data[cpref]
datal["file_names"].append( ffname )
for key in groups[gname]:
datal[ key] = groups[gname][ key ]
else:
print "oops2. globed more than 2 files"
sys.exit(1)
for ffname in fnames:
print ffname
if not os.path.exists(ffname) or not os.path.isfile(ffname):
print "file %s does not exists" % ffname
sys.exit(1)
data[ffname] = {}
#pp(data)
with open(CSV_OUT, 'w') as ofh:
outlines = []
for gname in sorted( data ):
ostr = gname
datal = data[gname]
for field in ['gtype', 'gsize', 'gmin', 'gmax', 'file_names']:
opart = ""
fieldl = datal[ field ]
if isinstance(fieldl, list):
opart += "," + ";".join( fieldl )
else:
opart += "," + str(fieldl)
ostr += opart
print ostr
outlines.append( ostr + "\n" )
ofh.write("#group name,library name,library size,library size min,library size max,file names\n")
ofh.writelines( outlines )
if __name__ == '__main__':
main()
print "\n\n"
print 'finished'
run.py 0 → 100755
+ 0
− 0
View file @ 6efceef6
This diff is collapsed.
run2.py 0 → 100755
+ 0
− 0
View file @ 6efceef6
This diff is collapsed.
sa1/run.py 0 → 120000
+ 2
− 0
View file @ 6efceef6
../run.py
\ No newline at end of file
sa1/sa.csv 0 → 100644
+ 9
− 0
View file @ 6efceef6
#group name,library name,library size,library size min,library size max,file names
120524_I238_FCD1174ACXX_L2_SZAXPI009264-133,pe,170,153,187,/home/assembly/tomato150/denovo/arcanum/raw/illumina/pairedend_170/120524_I238_FCD1174ACXX_L2_SZAXPI009264-133_2.fq.gz;/home/assembly/tomato150/denovo/arcanum/raw/illumina/pairedend_170/120524_I238_FCD1174ACXX_L2_SZAXPI009264-133_1.fq.gz
120527_I247_FCC0U0NACXX_L6_SZAXPI009264-133,pe,170,153,187,/home/assembly/tomato150/denovo/arcanum/raw/illumina/pairedend_170/120527_I247_FCC0U0NACXX_L6_SZAXPI009264-133_2.fq.gz;/home/assembly/tomato150/denovo/arcanum/raw/illumina/pairedend_170/120527_I247_FCC0U0NACXX_L6_SZAXPI009264-133_1.fq.gz
120527_I247_FCC0U0NACXX_L7_TOMxitDGJDWAAPEI-93,mp,2000,1800,2200,/home/assembly/tomato150/denovo/arcanum/raw/illumina/matepair_2000/120527_I247_FCC0U0NACXX_L7_TOMxitDGJDWAAPEI-93_1.fq.gz;/home/assembly/tomato150/denovo/arcanum/raw/illumina/matepair_2000/120527_I247_FCC0U0NACXX_L7_TOMxitDGJDWAAPEI-93_2.fq.gz
120530_I235_FCC0U06ACXX_L1_TOMxitDGJDWAAPEI-93,mp,2000,1800,2200,/home/assembly/tomato150/denovo/arcanum/raw/illumina/matepair_2000/120530_I235_FCC0U06ACXX_L1_TOMxitDGJDWAAPEI-93_1.fq.gz;/home/assembly/tomato150/denovo/arcanum/raw/illumina/matepair_2000/120530_I235_FCC0U06ACXX_L1_TOMxitDGJDWAAPEI-93_2.fq.gz
120530_I235_FCC0U06ACXX_L8_SZAXPI009264-133,pe,170,153,187,/home/assembly/tomato150/denovo/arcanum/raw/illumina/pairedend_170/120530_I235_FCC0U06ACXX_L8_SZAXPI009264-133_2.fq.gz;/home/assembly/tomato150/denovo/arcanum/raw/illumina/pairedend_170/120530_I235_FCC0U06ACXX_L8_SZAXPI009264-133_1.fq.gz
120530_I649_FCC0U6WACXX_L4_SZAXPI009359-46,pe,500,450,550,/home/assembly/tomato150/reseq/raw/058/illumina/pairedend_500/120530_I649_FCC0U6WACXX_L4_SZAXPI009359-46_1.fq.gz;/home/assembly/tomato150/reseq/raw/058/illumina/pairedend_500/120530_I649_FCC0U6WACXX_L4_SZAXPI009359-46_2.fq.gz
120615_I248_FCD14ADACXX_L7_TOMxitDGJDWAAPEI-93,mp,2000,1800,2200,/home/assembly/tomato150/denovo/arcanum/raw/illumina/matepair_2000/120615_I248_FCD14ADACXX_L7_TOMxitDGJDWAAPEI-93_2.fq.gz;/home/assembly/tomato150/denovo/arcanum/raw/illumina/matepair_2000/120615_I248_FCD14ADACXX_L7_TOMxitDGJDWAAPEI-93_1.fq.gz
120624_I232_FCC0U3HACXX_L5_TOMxitDGJDWAAPEI-93,mp,2000,1800,2200,/home/assembly/tomato150/denovo/arcanum/raw/illumina/matepair_2000/120624_I232_FCC0U3HACXX_L5_TOMxitDGJDWAAPEI-93_2.fq.gz;/home/assembly/tomato150/denovo/arcanum/raw/illumina/matepair_2000/120624_I232_FCC0U3HACXX_L5_TOMxitDGJDWAAPEI-93_1.fq.gz
sa1/sa.fa 0 → 120000
+ 2
− 0
View file @ 6efceef6
../../allpaths_lg_sample_arcanum_raw/sa/data/run/ASSEMBLIES/test/final.assembly.fasta
\ No newline at end of file
sa1/sa.groups.csv 0 → 120000
+ 2
− 0
View file @ 6efceef6
../../allpaths_lg_sample_arcanum_raw/in_groups2.csv
\ No newline at end of file
sa1/sa.lib.csv 0 → 120000
+ 2
− 0
View file @ 6efceef6
../../allpaths_lg_sample_arcanum_raw/in_libs2.csv
\ No newline at end of file
sa2/run.py 0 → 120000
+ 2
− 0
View file @ 6efceef6
../run.py
\ No newline at end of file
sa2/sa.fa 0 → 120000
+ 2
− 0
View file @ 6efceef6
../../allpaths_lg_sample_arcanum_raw/sa/data/run/ASSEMBLIES/test2/final.assembly.fasta
\ No newline at end of file
sa2/sa.groups.csv 0 → 120000
+ 2
− 0
View file @ 6efceef6
../../allpaths_lg_sample_arcanum_raw/in_groups2.csv
\ No newline at end of file
sa2/sa.libs.csv 0 → 120000
+ 2
− 0
View file @ 6efceef6
../../allpaths_lg_sample_arcanum_raw/in_libs2.csv
\ No newline at end of file
scons/scons-local-2.3.0/SCons/Action.py 0 → 100644
+ 0
− 0
View file @ 6efceef6
This diff is collapsed.
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment