util.py 2.38 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
import sqlalchemy
from sqlalchemy import MetaData, Table, Column, Integer, Date, Float, Text, create_engine

def take_first(iter):
    for i in iter:
        return i


def count_lines(files):
    """Checks the number of lines in the input CSV files.

    They should all be the same else throw an error.
    """
    print("Checking file row counts...")
    counts = {}
    for band, fname in files.items():
        with open(fname) as my_file:
            c = sum(1 for _ in my_file)
        counts[fname] = c
        print(f"  - {fname}: {c}")

    if len(set(counts.values())) > 1:
        msg = "CSV files do not have the same number of rows!"
        raise RuntimeError(msg)

    return take_first(counts.values())


def printProgressBar (iteration, total, prefix = '', suffix = '', decimals = 1, length = 100, fill = '█', printEnd = "\r"):
    """
    Call in a loop to create terminal progress bar
    @params:
        iteration   - Required  : current iteration (Int)
        total       - Required  : total iterations (Int)
        prefix      - Optional  : prefix string (Str)
        suffix      - Optional  : suffix string (Str)
        decimals    - Optional  : positive number of decimals in percent complete (Int)
        length      - Optional  : character length of bar (Int)
        fill        - Optional  : bar fill character (Str)
        printEnd    - Optional  : end character (e.g. "\r", "\r\n") (Str)
    """
    percent = ("{0:." + str(decimals) + "f}").format(100 * (iteration / float(total)))
    filledLength = int(length * iteration // total)
    bar = fill * filledLength + '-' * (length - filledLength)
    print('\r%s |%s| %s%% %s' % (prefix, bar, percent, suffix), end = printEnd)
    # Print New Line on Complete
    if iteration == total:
        print()


def prepare_db(dsn, table_name, bands, has_sensor_info):
    engine = create_engine(dsn)
    meta = MetaData(engine)
    tbl = Table(table_name, meta,
                Column('fieldID', Integer, primary_key=True, nullable=False),
                Column('day', Date, primary_key=True, nullable=False),
                )
    if has_sensor_info:
        tbl.append_column(Column('sensor', Text, primary_key=True, nullable=False))

    for col_name in bands:
        tbl.append_column(Column(col_name, Float, nullable=True))

    try:
        tbl.drop()
    except sqlalchemy.exc.OperationalError:
        pass
    tbl.create()

    return engine