studybehaviorAnalysisLivingTextbook/FilterData.py at master · Kyanite3221/studybehaviorAnalysisLivingTextbook · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
import datetime
import re

from openpyxl import *

import Main


# this method is highly inefficient and can be replaced by filterAndExtractData
def filterData(fileName, settings):
    wb = load_workbook(fileName)
    pageLoads = wb['Page loads']
    toRemove = []
    startPeriod = settings['period']['startPeriod']
    endPeriod = settings['period']['endPeriod']
    for row in pageLoads.iter_rows(min_row=2):
        if settings['period']['usePeriod']:
            if not (startPeriod <= row[Main.TIMESTAMPINDEX].value <= endPeriod):
                toRemove.append(row[1].row)
                continue

        if settings['students']['whitelist']:
            keep = False
            # students must be on the list to be taken into account
            for requirement in settings['students']['list']:
                if requirement['type'] == "regex":
                    matcher = re.compile(requirement['value'])
                    if matcher.match(row[Main.USERIDINDEX].value):
                        print("user may stay")
                        print(row[Main.USERIDINDEX].value)
                        keep = True
                        break
                elif requirement['value'] == row[Main.USERIDINDEX].value:
                    keep = True
                    print("user may stay")
                    print(row[Main.USERIDINDEX].value)
                    break
            if not keep:
                toRemove.append(row[1].row)

        else:
            for requirement in settings['students']['list']:
                if requirement['type'] == "regex":
                    matcher = re.compile(requirement['value'])
                    if matcher.match(row[Main.USERIDINDEX].value):
                        toRemove.append(row[1].row)
                        break
                elif requirement['value'] == row[Main.USERIDINDEX].value:
                    toRemove.append(row[1].row)
                    break

    toRemove = toRemove[::-1]
    print("done with reading")
    for i in toRemove:
        pageLoads.delete_rows(i)
    print("done removing")
    events = wb['Events']
    toRemove = []
    for row in events.iter_rows(min_row=2):
        if settings['period']['usePeriod']:
            if not (startPeriod <= row[Main.TIMESTAMPINDEX].value <= endPeriod):
                toRemove.append(row[1].row)
                continue

        if settings['students']['whitelist']:
            keep = False
            # students must be on the list to be taken into account
            for requirement in settings['students']['list']:
                if requirement['type'] == "regex":
                    matcher = re.compile(requirement['value'])
                    if matcher.match(row[Main.USERIDINDEX].value):
                        print("user may stay")
                        print(row[Main.USERIDINDEX].value)
                        keep = True
                        break
                elif requirement['value'] == row[Main.USERIDINDEX].value:
                    keep = True
                    print("user may stay")
                    print(row[Main.USERIDINDEX].value)
                    break
            if not keep:
                toRemove.append(row[1].row)

        else:
            for requirement in settings['students']['list']:
                if requirement['type'] == "regex":
                    matcher = re.compile(requirement['value'])
                    if matcher.match(row[Main.USERIDINDEX].value):
                        toRemove.append(row[1].row)
                        break
                elif requirement['value'] == row[Main.USERIDINDEX].value:
                    toRemove.append(row[1].row)
                    break
    print("done reading")
    for i in toRemove:
        events.delete_rows(i)
    wb.save(fileName)

def filterAndExtractData(fileNames, settings,debug=False):
    pageName = "Page loads"
    pageName2 = "Events"
    data = {pageName: [], pageName2: []}
    usePeriod = settings['period']['usePeriod']
    startPeriod = settings['period']['startDate'] if usePeriod else None
    endPeriod = settings['period']['endDate'] if usePeriod else None
    if type(fileNames) is list:
        if debug: print("multpiple files")
        for fileName in fileNames:
            wb = load_workbook(fileName)

            extractAndFilterData(data, endPeriod, pageName, settings, startPeriod, wb, debug)
            extractAndFilterData(data, endPeriod, pageName2, settings, startPeriod, wb, debug)
            wb.close()
            if debug: print("done with" + fileName)
    else:
        wb = load_workbook(fileNames)

        extractAndFilterData(data, endPeriod, pageName, settings, startPeriod, wb, debug)
        extractAndFilterData(data, endPeriod, pageName2, settings, startPeriod, wb, debug)
        wb.close()
    return data


def extractAndFilterData(data, endPeriod, pageName, settings, startPeriod, wb, debug):
    pageLoads = wb[pageName]
    for requirement in settings['students']['list']:
        requirement['value'] = re.compile(requirement['value']) if requirement['type'] == "regex" else requirement['value']

    for row in pageLoads.iter_rows(min_row=2):
        if settings['period']['usePeriod']:
            if not (startPeriod <= row[Main.TIMESTAMPINDEX].value <= endPeriod + datetime.timedelta(days=1)):
                continue  # to the next row

        if settings['students']['whitelist']:  # the given requirements are a whitelist
            for requirement in settings['students']['list']:
                if requirement['type'] == "regex":
                    matcher = requirement['value']
                    if matcher.match(row[Main.USERIDINDEX].value):
                        data[pageName].append(row)
                        break
                elif requirement['value'] == row[Main.USERIDINDEX].value:
                    data[pageName].append(row)
                    break

        else:  # the given requirements are a blacklist
            stay = True
            for requirement in settings['students']['list']:
                if requirement['type'] == "regex":
                    matcher = requirement['value']
                    if matcher.match(row[Main.USERIDINDEX].value):
                        stay = False
                        break
                elif requirement['value'] == row[Main.USERIDINDEX].value:
                    stay = False
                    break
            if stay:
                data[pageName].append(row)
    if debug: print("done with reading " + pageName)