from __future__ import print_function import sys import os sys.path.extend(['.', '..']) from pycparser import parse_file, c_ast from Queue import Queue from threading import Thread import files class Worker (Thread): """Thread executing tasks from a given tasks queue""" def __init__ (self, tasks): Thread.__init__(self) self.tasks = tasks self.daemon = True self.start() def run (self): while True: func, args, kargs = self.tasks.get() try: func(*args, **kargs) except Exception as e: print(e) self.tasks.task_done() class ThreadPool: """Pool of threads consuming tasks from a queue""" def __init__ (self, num_threads): self.tasks = Queue(num_threads) for _ in range(num_threads): Worker(self.tasks) def add_task (self, func, *args, **kargs): """Add a task to the queue""" self.tasks.put((func, args, kargs)) def wait_completion (self): """Wait for completion of all the tasks in the queue""" self.tasks.join() def processFile (filename): ast = parse_file(filename, use_cpp=True, cpp_path='gcc', cpp_args=['-E', r'-Iutils/fake_libc_include']) funcDefList = [node for (_, node) in ast.children() if node.__class__.__name__ is 'FuncDef' or node.__class__.__name__ is 'Def'] print("> AST size: %d" % len(funcDefList)) def processStudentData (studentData, assignment): print("Processing data from student %s at assignment %s" %(studentData[0], assignment)) try: processFile(studentData[1]) except: print("> failed to process file: %s" % studentData[1]) def loadStudentsFolders (raiz, folder): userDataPath = os.path.join(raiz, folder) return files.filesFromFolder(userDataPath, "usersdata") def loadStudentFiles (raiz, folder, studentsFolders): studentsData = [] for s in studentsFolders: path = os.path.join(raiz, folder, "usersdata", s) finalFolder = files.highestFileName(path) cFileFolder = os.path.join(path, finalFolder, "submittedfiles") cFiles = files.getFilesInFolder(cFileFolder, "*[cC]*") if len(cFiles) == 0: studentsData.append((s, "")) else: studentsData.append((s, cFiles[0])) return studentsData def loadAssignments (raiz): assignmentsFolders = files.filesFromFolder(raiz, "") assignments = {} for a in assignmentsFolders: studentsFolders = loadStudentsFolders(raiz, a) if(len(studentsFolders) == 0): assignments[a] = [] continue studentsData = loadStudentFiles(raiz, a, studentsFolders) assignments[a] = studentsData return assignments #--- run ---# if __name__ == "__main__": if len(sys.argv) > 1: raiz = "./" + sys.argv[1] data = loadAssignments(raiz) pool = ThreadPool(10) for a in data: for studentData in data[a]: pool.add_task(processStudentData, studentData, a) pool.wait_completion() print("waiting") # assigments = files.filesFromFolder(raiz, "") # for a in assigments: # userDataPath = os.path.join(raiz, a) # students = files.filesFromFolder(userDataPath, "usersdata") # if len(students) == 0: # continue # for s in students: # path = os.path.join(userDataPath, "usersdata", s) # finalFolder = files.highestFileName(path) # cFileFolder = os.path.join(path, finalFolder, "submittedfiles") # cFiles = files.getFilesInFolder(cFileFolder, "*.c") # for i in cFiles: # print(i) # pool = ThreadPool(20) # for file in inputs: # pool.add_task(processFile, file) # pool.wait_completion() else: print("No filename")