Commit 6efceef6 authored by Aflitos, Saulo Alves's avatar Aflitos, Saulo Alves
Browse files

first

parents
*.tar.gz
*.dblite
*.fai
*.pyc
*.dat
*.dot
*.png
*.html
/home/assembly/nobackup/metrics/FRC_align/src/FRC
\ No newline at end of file
run2.py
\ No newline at end of file
#!/usr/bin/python
import sys,os
import datetime
ignore_list = ['/bin/echo']
#['/mnt/nexenta/aflit001/nobackup/Data/', '']
replace = [ ['Data/', ''], ['out/', ''], ['data/', '']]
def parseStatus(status):
#01234567891
# E = exists
# R = exists in repository only
# b = implicit builder
# B = explicit builder
# S = side effect
#01234567891
# P = precious
# A = always build
# C = current
# N = no clean
# H = no cache
#01234567891
statusF = 0
if status[0] == "E": statusF += 1
if status[1] == "R": statusF += 2
if status[2] == "b": statusF += 4
if status[3] == "B": statusF += 4
if status[3] == "S": statusF += 8
if status[4] == "P": statusF += 16
if status[5] == "A": statusF += 32
if status[6] == "C": statusF += 64
if status[7] == "N": statusF += 128
if status[8] == "H": statusF += 256
return statusF
def clean_for_dot(expr):
expr = expr.strip()
for rep in replace:
#print "REPLACING",rep[0],'to',rep[1],'in',expr,
expr = expr.replace(rep[0],rep[1])
#print " BECOMING",expr
if expr[0] == '/': expr = os.path.basename( expr )
expr = expr.replace("/","\/").replace("-","\-").replace("_","\_")
if expr[0] == "[": expr = expr[1: ]
if expr[-1] == "]": expr = expr[:-1]
return expr
class node(object):
def __init__(self,path, status, depth):
self.path = path
self.status = status
self.depth = depth
self.children = []
self.statusF = parseStatus(self.status)
def add_child(self,child):
if not child in self.children: self.children.append(child)
def print_rec(self,depth):
print " "*depth,self.name
for c in self.children:
c.print_rec(depth+1)
def get_name(self):
return self.path
def get_label_short(self):
return self.path
def get_status(self):
return self.statusF
def get_depth(self):
return self.depth
def get_status_color(self):
color = 'white'
if (self.statusF % 2 ) != 0:
color = 'green'
return color
def is_fixed_tool(self):
return self.path in ignore_list
class scons():
def __init__(self):
self.stack = []
self.map = {}
def printOut(self):
o = open('SConstruct.tree.dot', 'w')
def wl(s): o.write("%s\n"%s)
wl("digraph G {")
##wl("""size="100x100" """)
##wl("""rankdir=TB """)
wl("raksep=equally")
#wl("nodesep=0.8")
wl("pagedir=BL")
wl("splines=true")
wl("concentrate=true")
wl("fontname=Helvetica")
wl("remincross=true")
wl("searchsize=100")
#wl("""rankdir=RL """)
#wl("ratio = 1")
#wl("ratio = compress")
wl("ratio = auto")
#Nodes:
for obj in self.map.values():
if not obj.is_fixed_tool():
#wl("""\"%s\" [label=\"%s\", fontsize=16, style=filled, fillcolor=%s]"""%(scons_obj.get_name(),scons_obj.get_label_short(),scons_obj.get_status_color()) )
#wl("""\"%s\" [label=\"%s\", nodesep=0.75, ranksep=0.75, shape=box, weight=1.2, fontsize=16, style=filled, fillcolor=%s]"""%(scons_obj.get_name(),scons_obj.get_label_short(),scons_obj.get_status_color()) )
wl("""\"%s\" [label=\"%s\", group=%s, rank=%s, shape=box, fontsize=24, style=filled, fillcolor=%s]"""%(obj.get_name(),obj.get_label_short(), obj.get_depth(), obj.get_depth(), obj.get_status_color()) )
#Connections:
for obj in self.map.values():
if not obj.is_fixed_tool() :
children = obj.children
if len(children) == 0:
continue
names = []
exts = []
for c in children:
if not c.is_fixed_tool():
name = c.get_name()
ext = os.path.splitext(name)
if len(ext) > 0:
ext = ext[1][1:]
names.append(name)
exts.append(ext)
nameext = zip(exts,names,children)
#sys.stderr.writelines("NAMEEXT 1"+str(nameext))
nameext.sort()
#sys.stderr.writelines("NAMEEXT 2"+str(nameext))
(sortedNames, sortedExts, sortedChildren) = zip( *nameext )
for c in sortedChildren:
if not c.is_fixed_tool():
wl("\"%s\" -> \"%s\""%(c.get_name(), obj.get_name() ) )
#wl("\"%s\" -> \"%s\""%(scons_obj.get_name(), c.get_name() ) )
wl("}")
def initial_filter(datalines):
return [d for d in datalines if d.strip() and d.strip()[0] in ["["] ]
class htmlTree():
def __init__(self):
self.lines = []
def checkRoot(self, str):
if str[0] == "[" and str[-1] == "]":
return False
else:
return True
def add(self, statusStr, pre, pos, nameClean, depth):
status = parseStatus(statusStr)
position = len(self.lines)
self.lines.append([status, pre, nameClean, depth, self.checkRoot(pos)])
def export(self):
lines = []
lines.append("<html>")
#www.perbang.dk/rgbgradient/ AD0000 E5E5FF 12 steps
lines.append("""
<style type="text/css">
p {
margin: 0px;
font-family: monospace;
white-space: nowrap;
display: inline;
}
p.parent { font-weight: 900 }
p.child { font-weight: 100 }
p.run { color: #00AA00 }
p.notrun { color: #FF0000 }
p { display: block }
</style>
<script src="http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
<script type="text/javascript">
function hide(name) {
$('.'+name).hide();
//alert('hiding '+name)
}
function show(name) {
$('.'+name).show();
//alert('showing '+name)
}
</script>
""")
#p.depth01 { background-color: #000000 }
#p.depth02 { background-color: #333333 }
#p.depth03 { background-color: #555555 }
#p.depth04 { background-color: #666666 }
#p.depth05 { background-color: #7A7A7A }
#p.depth06 { background-color: #8B8B8B }
#p.depth07 { background-color: #999999 }
#p.depth08 { background-color: #AAAAAA }
#p.depth09 { background-color: #BBBBBB }
#p.depth10 { background-color: #CCCCCC }
#p.depth11 { background-color: #DDDDDD }
#p.depth12 { background-color: #EEEEEE }
now = str(datetime.datetime.now())
print now
lines.append(" <body>")
lines.append("""
<table>
<tr>
<td>
Hide
</td>
<td>
<input type="button" onclick="hide('parent')" value="parent"/>
</td>
<td>
<input type="button" onclick="hide('child')" value="child"/>
</td>
<td>
<input type="button" onclick="hide('run')" value="run"/>
</td>
<td>
<input type="button" onclick="hide('notrun')" value="not run"/>
</td>
</tr>
<tr>
<td>
Show
</td>
<td>
<input type="button" onclick="show('parent')" value="parent"/>
</td>
<td>
<input type="button" onclick="show('child')" value="child"/>
</td>
<td>
<input type="button" onclick="show('run')" value="run"/>
</td>
<td>
<input type="button" onclick="show('notrun')" value="not run"/>
</td>
</tr>
</table>
""")
lines.append("LABEL:<br/>")
lines.append(' <p class="parent run" >'+'Finished Running - Original File'.replace(' ', '&nbsp;')+'</p>')
lines.append(' <p class="child run" >'+'Finished Running - Copy File'.replace(' ', '&nbsp;')+'</p>')
lines.append(' <p class="parent notrun" >'+'Not Run - Original File'.replace(' ', '&nbsp;')+'</p>')
lines.append(' <p class="child notrun" >'+'Not Run - Copy File'.replace(' ', '&nbsp;')+'</p>')
lines.append(' <p>'+now+'</p>')
#lines.append(' <p >Level:</p>')
#for p in range(1, 13):
# lines.append(' <p class="parent run depth%02d">&nbsp;%02d&nbsp;</p>' % (p ,p))
lines.append(' <br/><br/>')
for line in self.lines:
status = line[0]
pre = line[1]
name = line[2]
depth = line[3]
parent = line[4]
refname = name.replace('\/', '').replace('\\', '').replace('.', '')
name = name.replace('\\', '')
pre = pre.replace(' ', '&nbsp;')
linkStr = ''
parentHood = ''
runStatus = ''
if parent:
linkStr = '<a name="'+refname+'"></a>'
parentHood = 'parent'
else:
linkStr = '<a href="#'+refname+'">^</a>'
parentHood = 'child'
if (status % 2 ) != 0:
runStatus = 'run'
else:
runStatus = 'notrun'
lines.append(' <p class="%s %s depth%02d">' % (parentHood, runStatus, depth) + pre + linkStr + name + '</p>')
lines.append(" </body>")
lines.append("</html>")
f = open('SConstruct.tree.html', 'w')
f.writelines('\n'.join(lines))
f.close()
def main():
data = sys.stdin.readlines()
data = initial_filter(data)
HTML = htmlTree()
if len(sys.argv) > 1:
replace.append([sys.argv[1] + '/', ''])
replace.append([sys.argv[1] + '_', ''])
root = node("root",' '*9, 0)
scons_obj = scons()
scons_obj.stack.append( root )
def does_object_exist(name): return name in scons_obj.map.keys()
html = []
htmlKeys = {}
# Read the Data:
lc = 0
for l in data:
lc += 1
l = l.strip()
#print "LINE ",lc,"1",l
statusStr = l[1:10]
nameStr = l[11:]
nameClean = nameStr
depth = 1
for p in range(0, len(nameStr)):
if nameStr[p] not in ['\[', ' ', '-', '+', '|']:
#print "FOUND",nameStr[p]
pre = nameStr[0:p]
pos = nameStr[p:]
#pre = pre.replace("+",'').replace("-",'').replace("|",'') #Remove the "tree parts" from the text
depth = ( (p - 2)/2 ) + 1
nameClean = clean_for_dot(pos)
HTML.add(statusStr, pre, pos, nameClean, depth)
break
#print "LINE ",lc,"LINE",l,"STATUS STR",statusStr,"NAME STR",nameStr,"NAME CLEAN",nameClean,"DEPTH",depth
#if depth == 0: depth = 1
#print "LINE ",lc,"3 '",nodename,","
#if nodename[0] != "/": nodename = "./"+nodename
#print "LINE ",lc,"4 '",nodename,","
if not does_object_exist(nameClean):
#print "LINE ",lc,"5",nodename,"does not exists"
scons_obj.map[nameClean] = node(nameClean,statusStr,depth)
#print "LINE ",lc,"6",nodename,"node",node(nodename).get_name()
#depth = (len(l) - len(l.strip() ) +1)/2
#print "LINE ",lc,"7 DEPTH",depth
# If the depth is one higher than the depth of the stack,
# add as a child object.
if len(scons_obj.stack) == depth -1:
#print "LINE ",lc,"8 CHILD"
scons_obj.stack[-1].add_child( scons_obj.map[nameClean] )
scons_obj.stack.append( scons_obj.map[nameClean] )
# Otherwise pop the stack until we reach the parent... and add as a child
else:
#print "LINE ",lc,"9 NOT CHILD"
while( len(scons_obj.stack) > depth ) : scons_obj.stack.pop()
if len(scons_obj.stack) > 0:
scons_obj.stack[-1].add_child( scons_obj.map[nameClean] )
scons_obj.stack.append( scons_obj.map[nameClean] )
#print '-'*20
#root.print_rec(0)
HTML.export()
scons_obj.printOut()
main()
#!/bin/bash
LOGFILE="SConstruct.run.log."`date +%Y%m%d_%H%M%S`".out"
#echo "LOGFILE $LOGFILE"
python scons/scons.py -n --stack-size=1024 --diskcheck=none $@ 2>&1 | tee $LOGFILE
#!/bin/bash
LOGFILE="SConstruct.run.log."`date +%Y%m%d_%H%M%S`".out"
#echo "LOGFILE $LOGFILE"
python scons/scons.py --stack-size=1024 --diskcheck=none $@ 2>&1 | tee $LOGFILE
#!/bin/bash
rm SConstruct.tree.dot 2>/dev/null
rm SConstruct.tree.png 2>/dev/null
rm SConstruct.tree.gif 2>/dev/null
rm SConstruct.tree.html 2>/dev/null
bash SConstruct.run -n --tree=prune,status $@ > SConstruct.tree.dat
#,derived
cat SConstruct.tree.dat | python SConstruct.graph.py $@
#dot -Tgif tree.dot > tree.gif
#dot -Tpng tree.dot > tree.png
# | tee tree.dot | dot -Tgif > tree.gif
#./run -n --tree=status $@ | tee tree.dat
#cat tree.dat | ./graph.py | neato -Tpng > neato_rendering.png
#cat tree.dat | ./graph.py | fdp -Tpng > fdp_rendering.png
#cat tree.dat | ./graph.py | dot -Tgif > tree.gif
cat SConstruct.tree.dot | /usr/bin/dot -x -Tpng > SConstruct.tree.png
#cat tree.dat | ./graph.py | circo -Tpng > circo_rendering.png
#!/usr/bin/python
import os, sys
import glob
from pprint import pprint as pp
if len(sys.argv) < 4:
print "no libs given"
print sys.argv[0], "<IN CSV GROUP> <IN CSV LIB> <OUT FILE>"
sys.exit(1)
CSV_GROUP = sys.argv[1]
CSV_LIB = sys.argv[2]
CSV_OUT = sys.argv[3]
if (not os.path.exists( CSV_GROUP )) or ((CSV_LIB is not None) and (not os.path.exists(CSV_LIB))):
if (not os.path.exists( CSV_GROUP )):
print "csv group file %s does not exists" % CSV_GROUP
if ((CSV_LIB is not None) and (not os.path.exists(CSV_LIB))):
print "csv lib file %s does not exists" % CSV_LIB
print sys.arv[0], "<IN CSV GROUP> <IN CSV LIB>"
sys.exit(1)
if os.path.exists( CSV_OUT ):
print "output file %s already exists" % CSV_OUT
sys.exit( 1 )
def main():
data = {}
groups = {}
with open(CSV_LIB, 'r') as fhd:
line_count = 0
for line in fhd:
line_count += 1
if line_count == 1:
continue
if len(line.strip()) == 0 or line[0] == '#':
continue
#0 1 2 3 4 5 6 7 8 9 10 11
#library_name, project_name, organism_name, type , paired, frag_size, frag_stddev, insert_size, insert_stddev, read_orientation, genomic_start, genomic_end
#frags_PE_170, arcanum , sa , fragment, 1 , 170 , 17 , , , inward , 0 , 0
cels = line.split(',')
print "parsing line '%s'" % line.strip()
gname = cels[0].strip()
fsize = cels[5].strip()
fstdd = cels[6].strip()
isize = cels[7].strip()
istdd = cels[8].strip()
if gname not in data:
groups[gname] = {}
gtype = ""
gsize = 0
gmin = 0
gmax = 0
gmax2 = 0
if len(fsize) > 0:
gtype = 'pe'
gsize = int(fsize)
gmin = int(fsize) - int(fstdd)
gmax = int(fsize) + int(fstdd)
gmax2 = int(fsize) + (3*int(fstdd))
elif len(isize) > 0:
gtype = 'mp'
#data[gname]['type'] = 'pe'
gsize = int(isize)
gmin = int(isize) - int(istdd)
gmax = int(isize) + int(istdd)
gmax2 = int(isize) + (3*int(istdd))
else:
print line
print "count not find size"
sys.exit(1)
groups[gname]['gtype'] = gtype
groups[gname]['gsize'] = gsize
groups[gname]['gmin' ] = gmin
groups[gname]['gmax' ] = gmax
groups[gname]['gmax2'] = gmax2
with open(CSV_GROUP, 'r') as fhd:
line_count = 0
for line in fhd:
line_count += 1
if line_count == 1:
continue
if len(line.strip()) == 0 or line[0] == '#':
continue
#0
#file_name , group_name, library_name
#120527_I247_FCC0U0NACXX_L7_TOMxitDGJDWAAPEI-93_*.fq.gz, MP_2000_1 , jumps_MP_2000
cels = line.split(',')
print "parsing line '%s'" % line.strip()
fname = cels[0].strip()
gname = cels[2].strip()
fnames = glob.glob(fname)
if len(fnames) == 0:
print "error globing '%s'" % fname.strip()
print " line '%s'" % line.strip()
sys.exit(1)
cpref = os.path.basename( os.path.commonprefix(fnames) )
cpref = cpref.rstrip("_.-")
print cpref
if len(fnames) % 2 != 0:
print "oops. failed to glob"
sys.exit(1)
elif len(fnames) == 2:
data[cpref] = { "file_names": [] }
for ffname in fnames:
print ffname
if not os.path.exists(ffname) or not os.path.isfile(ffname):
print "file %s does not exists" % ffname
sys.exit(1)
datal = data[cpref]
datal["file_names"].append( ffname )
for key in groups[gname]:
datal[ key] = groups[gname][ key ]
else:
print "oops2. globed more than 2 files"
sys.exit(1)
for ffname in fnames:
print ffname
if not os.path.exists(ffname) or not os.path.isfile(ffname):
print "file %s does not exists" % ffname
sys.exit(1)
data[ffname] = {}
#pp(data)
with open(CSV_OUT, 'w') as ofh:
outlines = []
for gname in sorted( data ):
ostr = gname
datal = data[gname]
for field in ['gtype', 'gsize', 'gmin', 'gmax', 'file_names']:
opart = ""
fieldl = datal[ field ]
if isinstance(fieldl, list):
opart += "," + ";".join( fieldl )
else:
opart += "," + str(fieldl)
ostr += opart
print ostr
outlines.append( ostr + "\n" )
ofh.write("#group name,library name,library size,library size min,library size max,file names\n")
ofh.writelines( outlines )