first

.gitignore

+*.tar.gz
+*.dblite
+*.fai
+*.pyc
+*.dat
+*.dot
+*.png
+*.html

FRC

+/home/assembly/nobackup/metrics/FRC_align/src/FRC
\ No newline at end of file

SConstruct

+run2.py
\ No newline at end of file

SConstruct.graph.py

+#!/usr/bin/python
+
+import sys,os
+import datetime
+
+ignore_list = ['/bin/echo']
+#['/mnt/nexenta/aflit001/nobackup/Data/', '']
+replace     = [ ['Data/', ''], ['out/', ''], ['data/', '']]
+
+def parseStatus(status):
+    #01234567891
+    # E         = exists
+    #  R        = exists in repository only
+    #   b       = implicit builder
+    #   B       = explicit builder
+    #    S      = side effect
+    #01234567891
+    #     P     = precious
+    #      A    = always build
+    #       C   = current
+    #        N  = no clean
+    #         H = no cache
+    #01234567891
+    statusF = 0
+    if status[0] == "E": statusF += 1
+    if status[1] == "R": statusF += 2
+    if status[2] == "b": statusF += 4
+    if status[3] == "B": statusF += 4
+    if status[3] == "S": statusF += 8
+    if status[4] == "P": statusF += 16
+    if status[5] == "A": statusF += 32
+    if status[6] == "C": statusF += 64
+    if status[7] == "N": statusF += 128
+    if status[8] == "H": statusF += 256
+    return statusF
+
+def clean_for_dot(expr):
+    expr = expr.strip()
+    for rep in replace:
+        #print "REPLACING",rep[0],'to',rep[1],'in',expr,
+        expr = expr.replace(rep[0],rep[1])
+        #print " BECOMING",expr
+    if expr[0] == '/': expr = os.path.basename( expr )
+    expr = expr.replace("/","\/").replace("-","\-").replace("_","\_")
+    if expr[0]  == "[": expr = expr[1: ]
+    if expr[-1] == "]": expr = expr[:-1]
+
+    return expr
+
+class node(object):
+        def __init__(self,path, status, depth):
+                self.path     = path
+                self.status   = status
+                self.depth    = depth
+                self.children = []
+                self.statusF  = parseStatus(self.status)
+
+        def add_child(self,child):
+                if not child in self.children: self.children.append(child)
+
+        def print_rec(self,depth):
+                print " "*depth,self.name
+                for c in self.children:
+                        c.print_rec(depth+1)
+
+        def get_name(self):
+                return self.path
+
+        def get_label_short(self):
+                return self.path
+
+        def get_status(self):
+                return self.statusF
+
+        def get_depth(self):
+            return self.depth
+
+        def get_status_color(self):
+                color = 'white'
+                if (self.statusF % 2 ) != 0:
+                    color = 'green'
+                return color
+
+        def is_fixed_tool(self):
+                return self.path in ignore_list
+
+class scons():
+    def __init__(self):
+        self.stack = []
+        self.map   = {}
+
+    def printOut(self):
+
+        o = open('SConstruct.tree.dot', 'w')
+        def wl(s): o.write("%s\n"%s)
+
+
+        wl("digraph G {")
+        ##wl("""size="100x100" """)
+        ##wl("""rankdir=TB """)
+        wl("raksep=equally")
+        #wl("nodesep=0.8")
+        wl("pagedir=BL")
+        wl("splines=true")
+        wl("concentrate=true")
+        wl("fontname=Helvetica")
+        wl("remincross=true")
+        wl("searchsize=100")
+        #wl("""rankdir=RL """)
+        #wl("ratio = 1")
+        #wl("ratio = compress")
+        wl("ratio = auto")
+
+        #Nodes:
+        for obj in self.map.values():
+                if not obj.is_fixed_tool():
+                        #wl("""\"%s\" [label=\"%s\", fontsize=16, style=filled, fillcolor=%s]"""%(scons_obj.get_name(),scons_obj.get_label_short(),scons_obj.get_status_color()) )
+                        #wl("""\"%s\" [label=\"%s\", nodesep=0.75, ranksep=0.75, shape=box, weight=1.2, fontsize=16, style=filled, fillcolor=%s]"""%(scons_obj.get_name(),scons_obj.get_label_short(),scons_obj.get_status_color()) )
+                        wl("""\"%s\" [label=\"%s\", group=%s, rank=%s, shape=box, fontsize=24, style=filled, fillcolor=%s]"""%(obj.get_name(),obj.get_label_short(), obj.get_depth(), obj.get_depth(), obj.get_status_color()) )
+
+        #Connections:
+        for obj in self.map.values():
+            if not obj.is_fixed_tool() :
+                children = obj.children
+                if len(children) == 0:
+                    continue
+                names    = []
+                exts     = []
+                for c in children:
+                    if not c.is_fixed_tool():
+                        name = c.get_name()
+                        ext  = os.path.splitext(name)
+                        if len(ext) > 0:
+                            ext = ext[1][1:]
+                        names.append(name)
+                        exts.append(ext)
+                nameext = zip(exts,names,children)
+                #sys.stderr.writelines("NAMEEXT 1"+str(nameext))
+                nameext.sort()
+                #sys.stderr.writelines("NAMEEXT 2"+str(nameext))
+                (sortedNames, sortedExts, sortedChildren) = zip( *nameext )
+
+                for c in sortedChildren:
+                    if not c.is_fixed_tool():
+                            wl("\"%s\" -> \"%s\""%(c.get_name(), obj.get_name() ) )
+                            #wl("\"%s\" -> \"%s\""%(scons_obj.get_name(), c.get_name() ) )
+        wl("}")
+
+
+def initial_filter(datalines):
+        return [d for d in datalines if d.strip() and d.strip()[0] in ["["] ]
+
+class htmlTree():
+    def __init__(self):
+        self.lines = []
+
+    def checkRoot(self, str):
+        if str[0]  == "[" and str[-1] == "]":
+            return False
+        else:
+            return True
+
+    def add(self, statusStr, pre, pos, nameClean, depth):
+        status   = parseStatus(statusStr)
+        position = len(self.lines)
+        self.lines.append([status, pre, nameClean, depth, self.checkRoot(pos)])
+
+    def export(self):
+        lines = []
+        lines.append("<html>")
+        #www.perbang.dk/rgbgradient/ AD0000 E5E5FF 12 steps
+        lines.append("""
+  <style type="text/css">
+    p             {
+                    margin: 0px;
+                    font-family: monospace;
+                    white-space: nowrap;
+                    display: inline;
+    }
+
+
+    p.parent  { font-weight: 900          }
+    p.child   { font-weight: 100          }
+    p.run     { color: #00AA00            }
+    p.notrun  { color: #FF0000            }
+    p         { display: block            }
+
+  </style>
+
+  <script src="http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
+
+  <script type="text/javascript">
+    function hide(name) {
+        $('.'+name).hide();
+        //alert('hiding '+name)
+    }
+
+    function show(name) {
+        $('.'+name).show();
+        //alert('showing '+name)
+    }
+  </script>
+
+""")
+        #p.depth01 { background-color: #000000 }
+        #p.depth02 { background-color: #333333 }
+        #p.depth03 { background-color: #555555 }
+        #p.depth04 { background-color: #666666 }
+        #p.depth05 { background-color: #7A7A7A }
+        #p.depth06 { background-color: #8B8B8B }
+        #p.depth07 { background-color: #999999 }
+        #p.depth08 { background-color: #AAAAAA }
+        #p.depth09 { background-color: #BBBBBB }
+        #p.depth10 { background-color: #CCCCCC }
+        #p.depth11 { background-color: #DDDDDD }
+        #p.depth12 { background-color: #EEEEEE }
+
+        now = str(datetime.datetime.now())
+        print now
+        lines.append("  <body>")
+
+
+        lines.append("""
+    <table>
+        <tr>
+            <td>
+                Hide
+            </td>
+            <td>
+                <input type="button" onclick="hide('parent')" value="parent"/>
+            </td>
+            <td>
+                <input type="button" onclick="hide('child')" value="child"/>
+            </td>
+            <td>
+                <input type="button" onclick="hide('run')" value="run"/>
+            </td>
+            <td>
+                <input type="button" onclick="hide('notrun')" value="not run"/>
+            </td>
+        </tr>
+        <tr>
+            <td>
+                Show
+            </td>
+            <td>
+                <input type="button" onclick="show('parent')" value="parent"/>
+            </td>
+            <td>
+                <input type="button" onclick="show('child')" value="child"/>
+            </td>
+            <td>
+                <input type="button" onclick="show('run')" value="run"/>
+            </td>
+            <td>
+                <input type="button" onclick="show('notrun')" value="not run"/>
+            </td>
+        </tr>
+    </table>
+        """)
+
+
+        lines.append("LABEL:<br/>")
+        lines.append('    <p class="parent run"    >'+'Finished Running - Original File'.replace(' ', '&nbsp;')+'</p>')
+        lines.append('    <p class="child  run"    >'+'Finished Running - Copy File'.replace(' ', '&nbsp;')+'</p>')
+        lines.append('    <p class="parent notrun" >'+'Not Run          - Original File'.replace(' ', '&nbsp;')+'</p>')
+        lines.append('    <p class="child  notrun" >'+'Not Run          - Copy File'.replace(' ', '&nbsp;')+'</p>')
+        lines.append('    <p>'+now+'</p>')
+
+        #lines.append('    <p                    >Level:</p>')
+        #for p in range(1, 13):
+        #    lines.append('    <p class="parent run depth%02d">&nbsp;%02d&nbsp;</p>' % (p ,p))
+        lines.append('    <br/><br/>')
+
+        for line in self.lines:
+            status    = line[0]
+            pre       = line[1]
+            name      = line[2]
+            depth     = line[3]
+            parent    = line[4]
+            refname   = name.replace('\/', '').replace('\\', '').replace('.', '')
+            name      = name.replace('\\', '')
+
+            pre = pre.replace(' ', '&nbsp;')
+
+            linkStr    = ''
+            parentHood = ''
+            runStatus  = ''
+            if parent:
+                linkStr    = '<a name="'+refname+'"></a>'
+                parentHood = 'parent'
+            else:
+                linkStr    = '<a href="#'+refname+'">^</a>'
+                parentHood = 'child'
+
+            if (status % 2 ) != 0:
+                runStatus = 'run'
+            else:
+                runStatus = 'notrun'
+
+            lines.append('    <p class="%s %s depth%02d">' % (parentHood, runStatus, depth) + pre + linkStr + name + '</p>')
+
+
+        lines.append("  </body>")
+        lines.append("</html>")
+
+        f = open('SConstruct.tree.html', 'w')
+        f.writelines('\n'.join(lines))
+        f.close()
+
+
+
+
+def main():
+        data = sys.stdin.readlines()
+        data = initial_filter(data)
+        HTML = htmlTree()
+
+        if len(sys.argv) > 1:
+            replace.append([sys.argv[1] + '/', ''])
+            replace.append([sys.argv[1] + '_', ''])
+
+        root             = node("root",' '*9, 0)
+        scons_obj = scons()
+        scons_obj.stack.append( root )
+
+        def does_object_exist(name): return name in scons_obj.map.keys()
+        html     = []
+        htmlKeys = {}
+
+        # Read the Data:
+        lc = 0
+        for l in data:
+                lc += 1
+                l = l.strip()
+                #print "LINE ",lc,"1",l
+                statusStr = l[1:10]
+                nameStr   = l[11:]
+                nameClean = nameStr
+                depth     = 1
+                for p in range(0, len(nameStr)):
+                    if nameStr[p] not in ['\[', ' ', '-', '+', '|']:
+                        #print     "FOUND",nameStr[p]
+                        pre       = nameStr[0:p]
+                        pos       = nameStr[p:]
+                        #pre       = pre.replace("+",'').replace("-",'').replace("|",'')   #Remove the "tree parts" from the text
+                        depth     = ( (p - 2)/2 ) + 1
+                        nameClean = clean_for_dot(pos)
+                        HTML.add(statusStr, pre, pos, nameClean, depth)
+                        break
+                #print "LINE ",lc,"LINE",l,"STATUS STR",statusStr,"NAME STR",nameStr,"NAME CLEAN",nameClean,"DEPTH",depth
+
+                #if depth == 0: depth = 1
+                #print "LINE ",lc,"3 '",nodename,","
+                #if nodename[0] != "/": nodename = "./"+nodename
+                #print "LINE ",lc,"4 '",nodename,","
+
+
+                if not does_object_exist(nameClean):
+                        #print "LINE ",lc,"5",nodename,"does not exists"
+                        scons_obj.map[nameClean] = node(nameClean,statusStr,depth)
+                        #print "LINE ",lc,"6",nodename,"node",node(nodename).get_name()
+
+                #depth = (len(l) - len(l.strip() ) +1)/2
+                #print "LINE ",lc,"7 DEPTH",depth
+
+                # If the depth is one higher than the depth of the stack,
+                # add as a child object.
+                if len(scons_obj.stack) == depth -1:
+                        #print "LINE ",lc,"8 CHILD"
+                        scons_obj.stack[-1].add_child( scons_obj.map[nameClean] )
+                        scons_obj.stack.append( scons_obj.map[nameClean] )
+
+                # Otherwise pop the stack until we reach the parent... and add as a child
+                else:
+                        #print "LINE ",lc,"9 NOT CHILD"
+                        while( len(scons_obj.stack) > depth ) : scons_obj.stack.pop()
+
+                        if len(scons_obj.stack) > 0:
+                            scons_obj.stack[-1].add_child( scons_obj.map[nameClean] )
+                        scons_obj.stack.append(        scons_obj.map[nameClean] )
+                #print '-'*20
+
+
+        #root.print_rec(0)
+
+        HTML.export()
+        scons_obj.printOut()
+
+main()

SConstruct.run

+#!/bin/bash
+LOGFILE="SConstruct.run.log."`date +%Y%m%d_%H%M%S`".out"
+#echo "LOGFILE $LOGFILE"
+python scons/scons.py -n --stack-size=1024 --diskcheck=none $@ 2>&1 | tee $LOGFILE

SConstruct.runReal

+#!/bin/bash
+LOGFILE="SConstruct.run.log."`date +%Y%m%d_%H%M%S`".out"
+#echo "LOGFILE $LOGFILE"
+python scons/scons.py --stack-size=1024 --diskcheck=none $@ 2>&1 | tee $LOGFILE

SConstruct.tree.sh

+#!/bin/bash
+rm SConstruct.tree.dot  2>/dev/null
+rm SConstruct.tree.png  2>/dev/null
+rm SConstruct.tree.gif  2>/dev/null
+rm SConstruct.tree.html 2>/dev/null
+
+bash SConstruct.run -n --tree=prune,status $@ > SConstruct.tree.dat
+#,derived
+
+cat SConstruct.tree.dat | python SConstruct.graph.py $@
+
+#dot -Tgif tree.dot > tree.gif
+#dot -Tpng tree.dot > tree.png
+
+# | tee tree.dot | dot   -Tgif > tree.gif
+#./run -n --tree=status $@ | tee tree.dat
+#cat tree.dat | ./graph.py | neato -Tpng > neato_rendering.png
+#cat tree.dat | ./graph.py | fdp   -Tpng > fdp_rendering.png
+#cat tree.dat | ./graph.py | dot   -Tgif > tree.gif
+cat SConstruct.tree.dot  | /usr/bin/dot -x -Tpng > SConstruct.tree.png
+#cat tree.dat | ./graph.py | circo -Tpng > circo_rendering.png

merger.py

+#!/usr/bin/python
+import os, sys
+import glob
+from pprint import pprint as pp
+
+
+if len(sys.argv) < 4:
+    print "no libs given"
+    print sys.argv[0], "<IN CSV GROUP> <IN CSV LIB> <OUT FILE>"
+    sys.exit(1)
+
+
+CSV_GROUP  = sys.argv[1]
+CSV_LIB    = sys.argv[2]
+CSV_OUT    = sys.argv[3]
+
+if (not os.path.exists( CSV_GROUP )) or ((CSV_LIB is not None) and (not os.path.exists(CSV_LIB))):
+    if (not os.path.exists( CSV_GROUP )):
+        print "csv group file %s does not exists" % CSV_GROUP
+
+    if ((CSV_LIB is not None) and (not os.path.exists(CSV_LIB))):
+        print "csv lib file %s does not exists" % CSV_LIB
+
+    print sys.arv[0], "<IN CSV GROUP> <IN CSV LIB>"
+    sys.exit(1)
+
+if os.path.exists( CSV_OUT ):
+    print "output file %s already exists" % CSV_OUT
+    sys.exit( 1 )
+
+
+
+
+def main():
+    data   = {}
+    groups = {}
+
+    with open(CSV_LIB, 'r') as fhd:
+        line_count = 0
+        for line in fhd:
+            line_count += 1
+            if line_count == 1:
+                continue
+
+            if len(line.strip()) == 0 or line[0] == '#':
+                continue
+
+            #0             1             2              3         4       5          6            7            8              9                 10             11
+            #library_name, project_name, organism_name, type    , paired, frag_size, frag_stddev, insert_size, insert_stddev, read_orientation, genomic_start, genomic_end
+            #frags_PE_170, arcanum     , sa           , fragment, 1     , 170      , 17         ,            ,              , inward          , 0            ,           0
+
+            cels   = line.split(',')
+            print  "parsing line '%s'" % line.strip()
+            gname  = cels[0].strip()
+            fsize  = cels[5].strip()
+            fstdd  = cels[6].strip()
+            isize  = cels[7].strip()
+            istdd  = cels[8].strip()
+
+            if gname not in data:
+                groups[gname] = {}
+
+            gtype = ""
+            gsize = 0
+            gmin  = 0
+            gmax  = 0
+            gmax2 = 0
+
+            if len(fsize) > 0:
+                gtype = 'pe'
+                gsize = int(fsize)
+                gmin  = int(fsize) - int(fstdd)
+                gmax  = int(fsize) + int(fstdd)
+                gmax2 = int(fsize) + (3*int(fstdd))
+
+            elif len(isize) > 0:
+                gtype = 'mp'
+                #data[gname]['type'] = 'pe'
+                gsize = int(isize)
+                gmin  = int(isize) - int(istdd)
+                gmax  = int(isize) + int(istdd)
+                gmax2 = int(isize) + (3*int(istdd))
+
+            else:
+                print line
+                print "count not find size"
+                sys.exit(1)
+
+            groups[gname]['gtype'] = gtype
+            groups[gname]['gsize'] = gsize
+            groups[gname]['gmin' ] = gmin
+            groups[gname]['gmax' ] = gmax
+            groups[gname]['gmax2'] = gmax2
+
+
+
+    with open(CSV_GROUP, 'r') as fhd:
+        line_count = 0
+        for line in fhd:
+            line_count += 1
+            if line_count == 1:
+                continue
+
+            if len(line.strip()) == 0 or line[0] == '#':
+                continue
+
+            #0
+            #file_name                                             , group_name, library_name
+            #120527_I247_FCC0U0NACXX_L7_TOMxitDGJDWAAPEI-93_*.fq.gz, MP_2000_1 , jumps_MP_2000
+
+            cels   = line.split(',')
+            print  "parsing line '%s'" % line.strip()
+            fname  = cels[0].strip()
+            gname  = cels[2].strip()
+            fnames = glob.glob(fname)
+
+            if len(fnames) == 0:
+                print "error globing '%s'" % fname.strip()
+                print " line '%s'"         % line.strip()
+                sys.exit(1)
+
+            cpref = os.path.basename( os.path.commonprefix(fnames) )
+            cpref = cpref.rstrip("_.-")
+            print cpref
+
+            if len(fnames) % 2 != 0:
+                print "oops. failed to glob"
+                sys.exit(1)
+
+            elif len(fnames) == 2:
+                data[cpref] = { "file_names": [] }
+
+                for ffname in fnames:
+                    print ffname
+
+                    if not os.path.exists(ffname) or not os.path.isfile(ffname):
+                        print "file %s does not exists" % ffname
+                        sys.exit(1)
+
+                    datal = data[cpref]
+                    datal["file_names"].append( ffname )
+
+                    for key in groups[gname]:
+                        datal[ key] = groups[gname][ key ]
+
+            else:
+                print "oops2. globed more than 2 files"
+                sys.exit(1)
+                for ffname in fnames:
+                    print ffname
+
+                    if not os.path.exists(ffname) or not os.path.isfile(ffname):
+                        print "file %s does not exists" % ffname
+                        sys.exit(1)
+
+                    data[ffname] = {}
+
+    #pp(data)
+
+    with open(CSV_OUT, 'w') as ofh:
+        outlines = []
+        for gname in sorted( data ):
+            ostr = gname
+            datal = data[gname]
+            for field in ['gtype', 'gsize', 'gmin', 'gmax', 'file_names']:
+                opart  = ""
+                fieldl = datal[ field ]
+                if isinstance(fieldl, list):
+                    opart += "," + ";".join( fieldl )
+
+                else:
+                    opart += "," + str(fieldl)
+
+                ostr += opart
+            print ostr
+            outlines.append( ostr + "\n" )
+        ofh.write("#group name,library name,library size,library size min,library size max,file names\n")
+        ofh.writelines( outlines )
+
+if __name__ == '__main__':
+    main()
+    print "\n\n"
+    print 'finished'

run.py

+#!/usr/bin/python
+import os, sys
+import subprocess