import csv import sys import os import tkinter as tk from tkinter import messagebox import hashlib import pickle import filters.ieeeFilter as IEEE, filters.scopusFilter as Scopus, filters.webScienceFilter as WebOfScience # titulo, abstract, autores, arquivo&linha, DOI, palavras-chave,palavras-chave extras FINAL_CSV_HEADERS = ['titulo', 'abstract', 'autores', 'doi', 'keywords','extra_keywords', 'fonte'] def loadFiles (files): loadedFiles = [] for file in files: loadedFiles.append((csv.DictReader(open(file['path']), delimiter=file['separator']), file)) return loadedFiles def saveAcceptedCSV (data): file = open('./accepted.csv', mode='w') headers = data['header'] writer = csv.writer(file, delimiter=',') writer.writerow(headers) data.pop('header') for row in data.values(): writer.writerow(row) file.flush() file.close() def saveRejectedCSV (data): file = open('./rejected.csv', mode='w') headers = data['header'] writer = csv.writer(file, delimiter=',') writer.writerow(headers) data.pop('header') for row in data.values(): writer.writerow(row) file.flush() file.close() def writeFinalLog (data): file = open("./log.txt", mode='w') file.write("accepted: {0}\nrejected: {1}\nduplicated: {2}\n".format(data['accepted'],data['rejected'],data['duplicate'])) file.flush() file.close() def removeStateFile (): try: os.remove('./.csvstate') except Exception: pass def saveState (data): file = open('./.csvstate', mode='wb') pickle.dump(data, file) file.close() def loadState (root): try: file = open('./.csvstate', mode='rb') answer = tk.messagebox.askyesno("State lading","Load last session state?") if answer: data = pickle.load(file) file.close() return data except Exception: pass return {'file_index':0, 'row_index':0, 'accepted':0, 'rejected':0, 'duplicate': 0} def loadConfigFile (): files = list() try: file = open('./csvconfig.txt') rawData = file.read() lines = rawData.split('\n') for line in lines: parameters = line.split(':') separator = parameters[2].encode('utf-8').decode("unicode_escape") fileConfig = {'path': parameters[0], 'filter': parameters[1], 'separator': separator} files.append(fileConfig) file.close() return files except FileNotFoundError: return list() def getFilter (file): if file['filter'] == 'ieee': return IEEE.IEEEFilter() elif file['filter'] == 'scopus': return Scopus.ScopusFilter() elif file['filter'] == 'webscience': return WebOfScience.WebScience() else: raise Exception('Undefined filter '+file['filter']) def restoreAcceptedCSV (): try: file = open('./accepted.csv') rows = list(csv.reader(file)) data = {} if len(rows) == 0: return data data['header'] = rows[0] for i in range(1,len(rows)): digest = hashlib.sha256(str.encode(rows[i][0])).hexdigest() data[digest] = rows[i] return data except FileNotFoundError: return {} def restoreRejectedCSV (): try: file = open('./rejected.csv') rows = list(csv.reader(file)) data = {} if len(rows) == 0: return data data['header'] = rows[0] for i in range(1,len(rows)): digest = hashlib.sha256(str.encode(rows[i][0])).hexdigest() data[digest] = rows[i] return data except FileNotFoundError: return {}