pHagen/UCS_Import_Python
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

updated project structure

Browse Source
This commit is contained in:
Patrick vom Hagen
2024-01-26 15:41:22 +01:00
parent 6ea62d15d7
commit 3175327cf2
24 changed files with 89 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
src/__init__.py Normal file
View File

140
src/compareL.py
View File

@@ -1,140 +0,0 @@
import csv
from Levenshtein import distance
import pandas as pd
import uuid
# TODO Filter für Spalten, ggfs. Klasse benötigt
# TODO Filter für Dublikate hier wird dann die Klasse benötigt
def read_csv(file_path):
data = []
with open(file_path, newline='', encoding='utf-8') as csvfile:
reader = csv.reader(csvfile, delimiter=';')
for row in reader:
data.append((row[0].strip(), row[1].strip()))
return data
def similar_sets(pair, data):
similar_pairs = []
for item in data:
if distance(pair[0], item[0]) <= 1 and distance(pair[1], item[1]) <= 1:
similar_pairs.append(item)
return similar_pairs
def compare_csv(file1, file2):
data1 = read_csv(file1)
data2 = read_csv(file2)
common_pairs = set(data1) & set(data2)
unique_pairs1 = set(data1) - common_pairs
unique_pairs2 = set(data2) - common_pairs
return common_pairs, unique_pairs1, unique_pairs2, data1, data2
def find_similar_pairs(pair, other_data):
similar_pairs = []
for item in other_data:
if distance(pair[0], item[0]) <= 2 and distance(pair[1], item[1]) <= 2:
similar_pairs.append(item)
return similar_pairs
def create_uuid():
return str(uuid.uuid4())
def add_hl_tag(row):
klasse = str(row['klasse']).lstrip('0')
return 'HL0707104-' + klasse
def create_import_list(path, path_new, old_pairs, new_pairs, common_pairs):
system_data = pd.read_csv(path, sep=';', encoding='utf-8')
system_data = system_data[~system_data[['name', 'vorname']].apply(tuple, axis=1).isin(old_pairs)]
# print(len(system_data))
# print(system_data)
new_data = pd.read_csv(path_new, sep=';', encoding='utf-8')
# Bei Schüler: alte Klassen gelöscht, mit neuen Klassen aus new-data auffüllen
matches = new_data[~new_data[['name', 'vorname']].apply(tuple, axis=1).isin(new_pairs)]
# matches.loc[:, 'klasse'] = matches.apply(add_hl_tag, axis=1)
# print(len(matches))
# print(matches)
system_data = pd.merge(system_data, matches, how='outer', left_on=['name', 'vorname'], right_on=['name', 'vorname'])
system_data = system_data[['name', 'vorname', 'klasse', 'schuelerid']]
system_data = system_data.drop('klasse', axis=1, errors='ignore')
print(system_data.columns)
print("Passende Einträge:" + str(len(system_data)))
new_data = new_data[~new_data[['name', 'vorname']].apply(tuple, axis=1).isin(common_pairs)]
# new_data = new_data.drop('Unnamed: 2', axis=1, errors='ignore')
new_uuids = []
for row in range(len(new_data)):
new_uuids.append(create_uuid())
new_data.insert(loc=2, column='schuelerid', value=new_uuids)
mailUserQuota = 2048
oxUserQuota = 20480
oxContext = 25
print(new_data.columns)
print("New Data:" + str(len(new_data)))
# vor dem merge daten ergänzen
# import_df = pd.merge(system_data, new_data, how='outer', left_on=['name', 'vorname', 'mailUserQuota', 'oxUserQuota', 'oxContext'], right_on=['name', 'vorname', 'mailUserQuota', 'oxUserQuota', 'oxContext'])
# import_df = pd.merge(system_data, new_data, how='outer',
# left_on=['name', 'vorname', 'klasse', 'schuelerid', 'mailUserQuota', 'oxUserQuota',
# 'oxContext'],
# right_on=['name', 'vorname', 'klasse', 'schuelerid', 'mailUserQuota', 'oxUserQuota',
# 'oxContext'])
import_df = pd.merge(system_data, new_data, how='outer',
left_on=['name', 'vorname', 'schuelerid'],
right_on=['name', 'vorname', 'schuelerid'])
import_df['mailUserQuota'] = mailUserQuota
import_df['oxUserQuota'] = oxUserQuota
import_df['oxContext'] = oxContext
import_df['klasse'] = None
import_df = import_df[['name', 'vorname', 'klasse', 'schuelerid', 'mailUserQuota', 'oxUserQuota', 'oxContext']]
# pd.set_option('display.max_rows', None)
# pd.set_option('display.max_columns', None)
# print(import_df)
# pd.reset_option('display.max_rows')
# pd.reset_option('display.max_columns')
out_path = '../GPS/outputLehrer.csv'
import_df.to_csv(out_path, sep=';', index=False)
print(len(import_df))
print('Lehrer Import Liste erzeugt')
print('Testuser manuell nachtragen!!')
def main():
file1_path = '../GPS/gpsLnew.csv'
file2_path = '../GPS/gpsLold2.cvs'
file3_path = '../GPS/gpsLold.csv'
common_pairs, new_pairs, old_pairs, data1, data2 = compare_csv(file1_path, file2_path)
print(f"Anzahl der übereinstimmenden Paare: {len(common_pairs)}")
print(f"Anzahl der neuen Einträge: {len(new_pairs)}")
print(f"Anzahl der veralteten Einträge: {len(old_pairs)}")
# Paare nur aus nicht zugeordneten Paaren aus neuer Liste erstellen
for pair in data1:
similar_pairs_list2 = find_similar_pairs(pair, set(data2) - {pair})
if similar_pairs_list2:
print(f"Ähnliche Paare in neuer Liste {pair} aktuell im System: {similar_pairs_list2}")
create_import_list(file3_path, file1_path, old_pairs, new_pairs, common_pairs)
if __name__ == "__main__":
main()

141
src/compareS.py
View File

@@ -1,141 +0,0 @@
import csv
from Levenshtein import distance
import pandas as pd
import uuid
# TODO Filter für Spalten, ggfs. Klasse benötigt
# TODO Filter für Dublikate hier wird dann die Klasse benötigt
def read_csv(file_path):
data = []
with open(file_path, newline='', encoding='utf-8') as csvfile:
reader = csv.reader(csvfile, delimiter=';')
for row in reader:
data.append((row[0].strip(), row[1].strip()))
return data
def similar_sets(pair, data):
similar_pairs = []
for item in data:
if distance(pair[0], item[0]) <= 1 and distance(pair[1], item[1]) <= 1:
similar_pairs.append(item)
return similar_pairs
def compare_csv(file1, file2):
data1 = read_csv(file1)
data2 = read_csv(file2)
common_pairs = set(data1) & set(data2)
unique_pairs1 = set(data1) - common_pairs
unique_pairs2 = set(data2) - common_pairs
return common_pairs, unique_pairs1, unique_pairs2, data1, data2
def find_similar_pairs(pair, other_data):
similar_pairs = []
for item in other_data:
if distance(pair[0], item[0]) <= 2 and distance(pair[1], item[1]) <= 2:
similar_pairs.append(item)
return similar_pairs
def create_uuid():
return str(uuid.uuid4())
def add_hl_tag(row):
klasse = str(row['klasse']).lstrip('0')
return 'HL0707104-' + klasse
def create_import_list(path, path_new, old_pairs, new_pairs, common_pairs):
system_data = pd.read_csv(path, sep=';', encoding='utf-8')
system_data = system_data[~system_data[['name', 'vorname']].apply(tuple, axis=1).isin(old_pairs)]
system_data = system_data.drop('username', axis=1, errors='ignore')
system_data = system_data.drop('klasse', axis=1, errors='ignore')
# print(len(system_data))
# print(system_data)
new_data = pd.read_csv(path_new, sep=';', encoding='utf-8')
# Bei Schüler: alte Klassen gelöscht, mit neuen Klassen aus new-data auffüllen
matches = new_data[~new_data[['name', 'vorname']].apply(tuple, axis=1).isin(new_pairs)]
matches.loc[:, 'klasse'] = matches.apply(add_hl_tag, axis=1)
# print(len(matches))
# print(matches)
system_data = pd.merge(system_data, matches, how='outer', left_on=['name', 'vorname'], right_on=['name', 'vorname'])
system_data = system_data[['name', 'vorname', 'klasse', 'schuelerid']]
# print(system_data)
print(len(system_data))
new_data = new_data[~new_data[['name', 'vorname']].apply(tuple, axis=1).isin(common_pairs)]
# new_data = new_data.drop('Unnamed: 2', axis=1, errors='ignore')
new_uuids = []
for row in range(len(new_data)):
new_uuids.append(create_uuid())
# Klasse?? Unterschied zwischen Lehrer und Schüler
# new_data['klasse'] = None
new_data.loc[:, 'klasse'] = new_data.apply(add_hl_tag, axis=1)
new_data.insert(loc=2, column='schuelerid', value=new_uuids)
mailUserQuota = 1024
oxUserQuota = 5120
oxContext = 25
# print(new_data)
print(len(new_data))
# vor dem merge daten ergänzen
# import_df = pd.merge(system_data, new_data, how='outer', left_on=['name', 'vorname', 'mailUserQuota', 'oxUserQuota', 'oxContext'], right_on=['name', 'vorname', 'mailUserQuota', 'oxUserQuota', 'oxContext'])
# import_df = pd.merge(system_data, new_data, how='outer',
# left_on=['name', 'vorname', 'klasse', 'schuelerid', 'mailUserQuota', 'oxUserQuota',
# 'oxContext'],
# right_on=['name', 'vorname', 'klasse', 'schuelerid', 'mailUserQuota', 'oxUserQuota',
# 'oxContext'])
import_df = pd.merge(system_data, new_data, how='outer',
left_on=['name', 'vorname', 'klasse', 'schuelerid'],
right_on=['name', 'vorname', 'klasse', 'schuelerid'])
import_df['mailUserQuota'] = mailUserQuota
import_df['oxUserQuota'] = oxUserQuota
import_df['oxContext'] = oxContext
# pd.set_option('display.max_rows', None)
# pd.set_option('display.max_columns', None)
# print(import_df)
# pd.reset_option('display.max_rows')
# pd.reset_option('display.max_columns')
out_path = '../GPS/outputSchueler.csv'
import_df.to_csv(out_path, sep=';', index=False)
print(len(import_df))
print('Schüler Import Liste erzeugt')
print('Testuser manuell nachtragen!!')
def main():
file1_path = '../GPS/gpsSnew.csv'
file2_path = '../GPS/gpsSold2.cvs'
file3_path = '../GPS/gpsSold.csv'
common_pairs, new_pairs, old_pairs, data1, data2 = compare_csv(file1_path, file2_path)
print(f"Anzahl der übereinstimmenden Paare: {len(common_pairs)}")
print(f"Anzahl der neuen Einträge: {len(new_pairs)}")
print(f"Anzahl der veralteten Einträge: {len(old_pairs)}")
# Paare nur aus nicht zugeordneten Paaren aus neuer Liste erstellen
# for pair in data1:
# similar_pairs_list2 = find_similar_pairs(pair, set(data2) - {pair})
#
# if similar_pairs_list2:
# print(f"Ähnliche Paare in neuer Liste {pair} aktuell im System: {similar_pairs_list2}")
create_import_list(file3_path, file1_path, old_pairs, new_pairs, common_pairs)
if __name__ == "__main__":
main()

70
src/format_csv.py
View File

@@ -1,70 +0,0 @@
import pandas as pd
import chardet
import csv
def check_file(path):
with open(path, 'rb') as file:
result = chardet.detect(file.read())
detected_encoding = result['encoding']
try:
pd.read_csv(path, encoding=detected_encoding)
except pd.errors.ParserError as e:
# Wenn ein Parserfehler auftritt, gibt eine Fehlermeldung aus
print(f"Fehler beim Einlesen der CSV-Datei: {e}")
print()
data = open(path, "r")
data = ''.join([i for i in data]).replace(",", "")
x = open(path, "w")
x.writelines(data)
x.close()
print(f"Alle Kommas entfernt")
# Prüft Formatierung der CSV, formatiert diese zu utf-8 und speichert das Ergebnis als neue Liste
def format_csv(path, type):
with open(path, 'rb') as file:
result = chardet.detect(file.read())
detected_encoding = result['encoding']
# CSV-Datei mit Pandas einlesen
try:
df = pd.read_csv(path, encoding=detected_encoding)
print("Datei erfolgreich eingelesen.")
df.to_csv(type, index=False, encoding='utf-8')
print("UTF-8 Kopie erfolgreich erstellt.")
except pd.errors.ParserError as e:
# Wenn ein Parserfehler auftritt, gibt eine Fehlermeldung aus
print(f"Fehler beim Einlesen der CSV-Datei: {e}")
def clean_data(path, clean):
try:
# Lese den Header der CSV-Datei
with open(path, 'r', newline='', encoding='utf-8') as csvfile:
reader = csv.reader(csvfile, delimiter=';')
header = next(reader)
# Finde die Indizes der Spalten 'Name' und 'Vorname' und 'Klasse'
name_index = header.index('name')
vorname_index = header.index('vorname')
klasse_index = header.index('klasse')
# Öffne die CSV-Datei im Schreibmodus und schreibe nur die gewünschten Spalten zurück
with open(clean, 'w', newline='', encoding='utf-8') as csvfile2:
writer = csv.writer(csvfile2, delimiter=';')
# Schreibe den neuen Header mit 'Name' und 'Vorname'
writer.writerow(['name', 'vorname', 'klasse'])
print(name_index, vorname_index, klasse_index)
for row in reader:
writer.writerow([row[name_index], row[vorname_index], row[klasse_index]])
print(f'Nur die Spalten "Name" und "Vorname" in der CSV-Datei {path} wurden beibehalten.')
except FileNotFoundError:
print(f'Die Datei {path} wurde nicht gefunden.')
except ValueError:
print(f'Die Spalten "Name" und "Vorname" wurden nicht gefunden.')

38
src/output.py
View File

@@ -1,38 +0,0 @@
import csv
# Zielformat: ${nachname};${vorname};HL070${SCHOOL}-${klasse};${recordID};1024;${ox_quota};${ox_context}
def create_output(path):
schule = input('Schule: ')
record_id = input('Record ID: ')
mail_quota = input('Mail')
ox_quota = input('OX Quota: ')
ox_context = input('OX Context: ')
data = []
with open(path, newline='', encoding='utf-8') as csvfile:
reader = csv.reader(csvfile, delimiter=';')
for row in reader:
data.append((row[0].strip(), row[1].strip(), schule, record_id, mail_quota, ox_quota, ox_context))
csv_file_path = '../Data/output.csv'
with open(csv_file_path, 'w', newline='', encoding='utf-8') as csvfile:
csv_writer = csv.writer(csvfile, delimiter=';')
# Schreibe die Header-Zeile (optional)
# name;vorname;klasse;schuelerid;mailUserQuota;oxUserQuota;oxContext
csv_writer.writerow(['name', 'vorname', 'klasse', 'schuelerid', 'mailUserQuota', 'oxUserQuota', 'oxContext'])
# TODO UUID prüfen bzw generien
# Schreibe die Daten aus dem Array in die CSV-Datei
csv_writer.writerows(data)
print(f"CSV-Datei wurde erfolgreich erstellt: {csv_file_path}")
if __name__ == "__main__":
create_output('../Data/test_new.csv')
# TODO Leerzeilen löschen
# TODO Klassenname umformatieren - HL070**** Nummer einfügen

22
src/restart.py
View File

@@ -1,22 +0,0 @@
########################################
# -- Python Script für UCS Import -- #
# -- by Patrick vom Hagen 2024 -- #
########################################
# IMPORT - standard Python imports für benötigte Bibliotheken #
import pandas as pd # pandas für Datenmanagement
import chardet # chardet erkennt Formatierung - Umwandlung des erkannten Formats in UTF-8
import csv # zur Verarbeitung von .csv Dateien
import uuid # zur Generierung von neuen UUIDs
# Flags / globale Variablen #
del_zeros = False # Boolean, ob führende Nullen bei Klassen entfernt werden sollen
get_typos = False # Boolean, ob geringe Unterschiede zwischen den Listen ausgegeben werden sollen
ox_context = 0 # OX Context pro Schule
mail_quota_lul = 2048 # MailUserQuota LuL
ox_quota_lul = 20480 # oxUserQuota LuL
mail_quota_sus = 1024 # MailUserQuota SuS
ox_quota_sus = 5120 # oxUserQuota LuL

0
src/step1.py Normal file
View File