From 3175327cf25a77d518a6a1713eeb820a95e88539 Mon Sep 17 00:00:00 2001 From: Patrick vom Hagen Date: Fri, 26 Jan 2024 15:41:22 +0100 Subject: [PATCH] updated project structure --- {GGS => Data/GGS}/Import_KuK.csv | 0 {GGS => Data/GGS}/downloadL.csv | 0 {GGS => Data/GGS}/downloadS.csv | 0 {GGS => Data/GGS}/download_intern.csv | 0 {GGS => Data/GGS}/export_schueler.csv | 0 {GGS => Data/GGS}/ggsLnew.csv | 0 {GGS => Data/GGS}/ggsLold.csv | 0 {GGS => Data/GGS}/ggsLold2.cvs | 0 {GGS => Data/GGS}/ggsSnew.csv | 0 {GGS => Data/GGS}/ggsSold.csv | 0 {GGS => Data/GGS}/ggsSold2.cvs | 0 {GGS => Data/GGS}/lehrer_HL0707104.csv | 0 {GGS => Data/GGS}/outputLehrer.csv | 0 {GGS => Data/GGS}/outputSchueler.csv | 0 {GGS => Data/GGS}/schueler_HL0707104.csv | 0 {src => first_try}/compareL.py | 8 +-- {src => first_try}/compareS.py | 8 +-- {src => first_try}/format_csv.py | 0 first_try/main.py | 26 +++++++++ {src => first_try}/output.py | 0 main.py | 74 ++++++++++++++++++------ src/__init__.py | 0 src/restart.py | 22 ------- src/step1.py | 0 24 files changed, 89 insertions(+), 49 deletions(-) rename {GGS => Data/GGS}/Import_KuK.csv (100%) rename {GGS => Data/GGS}/downloadL.csv (100%) rename {GGS => Data/GGS}/downloadS.csv (100%) rename {GGS => Data/GGS}/download_intern.csv (100%) rename {GGS => Data/GGS}/export_schueler.csv (100%) rename {GGS => Data/GGS}/ggsLnew.csv (100%) rename {GGS => Data/GGS}/ggsLold.csv (100%) rename {GGS => Data/GGS}/ggsLold2.cvs (100%) rename {GGS => Data/GGS}/ggsSnew.csv (100%) rename {GGS => Data/GGS}/ggsSold.csv (100%) rename {GGS => Data/GGS}/ggsSold2.cvs (100%) rename {GGS => Data/GGS}/lehrer_HL0707104.csv (100%) rename {GGS => Data/GGS}/outputLehrer.csv (100%) rename {GGS => Data/GGS}/outputSchueler.csv (100%) rename {GGS => Data/GGS}/schueler_HL0707104.csv (100%) rename {src => first_try}/compareL.py (96%) rename {src => first_try}/compareS.py (96%) rename {src => first_try}/format_csv.py (100%) create mode 100644 first_try/main.py rename {src => first_try}/output.py (100%) create mode 100644 src/__init__.py delete mode 100644 src/restart.py create mode 100644 src/step1.py diff --git a/GGS/Import_KuK.csv b/Data/GGS/Import_KuK.csv similarity index 100% rename from GGS/Import_KuK.csv rename to Data/GGS/Import_KuK.csv diff --git a/GGS/downloadL.csv b/Data/GGS/downloadL.csv similarity index 100% rename from GGS/downloadL.csv rename to Data/GGS/downloadL.csv diff --git a/GGS/downloadS.csv b/Data/GGS/downloadS.csv similarity index 100% rename from GGS/downloadS.csv rename to Data/GGS/downloadS.csv diff --git a/GGS/download_intern.csv b/Data/GGS/download_intern.csv similarity index 100% rename from GGS/download_intern.csv rename to Data/GGS/download_intern.csv diff --git a/GGS/export_schueler.csv b/Data/GGS/export_schueler.csv similarity index 100% rename from GGS/export_schueler.csv rename to Data/GGS/export_schueler.csv diff --git a/GGS/ggsLnew.csv b/Data/GGS/ggsLnew.csv similarity index 100% rename from GGS/ggsLnew.csv rename to Data/GGS/ggsLnew.csv diff --git a/GGS/ggsLold.csv b/Data/GGS/ggsLold.csv similarity index 100% rename from GGS/ggsLold.csv rename to Data/GGS/ggsLold.csv diff --git a/GGS/ggsLold2.cvs b/Data/GGS/ggsLold2.cvs similarity index 100% rename from GGS/ggsLold2.cvs rename to Data/GGS/ggsLold2.cvs diff --git a/GGS/ggsSnew.csv b/Data/GGS/ggsSnew.csv similarity index 100% rename from GGS/ggsSnew.csv rename to Data/GGS/ggsSnew.csv diff --git a/GGS/ggsSold.csv b/Data/GGS/ggsSold.csv similarity index 100% rename from GGS/ggsSold.csv rename to Data/GGS/ggsSold.csv diff --git a/GGS/ggsSold2.cvs b/Data/GGS/ggsSold2.cvs similarity index 100% rename from GGS/ggsSold2.cvs rename to Data/GGS/ggsSold2.cvs diff --git a/GGS/lehrer_HL0707104.csv b/Data/GGS/lehrer_HL0707104.csv similarity index 100% rename from GGS/lehrer_HL0707104.csv rename to Data/GGS/lehrer_HL0707104.csv diff --git a/GGS/outputLehrer.csv b/Data/GGS/outputLehrer.csv similarity index 100% rename from GGS/outputLehrer.csv rename to Data/GGS/outputLehrer.csv diff --git a/GGS/outputSchueler.csv b/Data/GGS/outputSchueler.csv similarity index 100% rename from GGS/outputSchueler.csv rename to Data/GGS/outputSchueler.csv diff --git a/GGS/schueler_HL0707104.csv b/Data/GGS/schueler_HL0707104.csv similarity index 100% rename from GGS/schueler_HL0707104.csv rename to Data/GGS/schueler_HL0707104.csv diff --git a/src/compareL.py b/first_try/compareL.py similarity index 96% rename from src/compareL.py rename to first_try/compareL.py index d4fdac0..df4e7f5 100644 --- a/src/compareL.py +++ b/first_try/compareL.py @@ -108,7 +108,7 @@ def create_import_list(path, path_new, old_pairs, new_pairs, common_pairs): # print(import_df) # pd.reset_option('display.max_rows') # pd.reset_option('display.max_columns') - out_path = '../GPS/outputLehrer.csv' + out_path = '../Data/GPS/outputLehrer.csv' import_df.to_csv(out_path, sep=';', index=False) print(len(import_df)) print('Lehrer Import Liste erzeugt') @@ -116,9 +116,9 @@ def create_import_list(path, path_new, old_pairs, new_pairs, common_pairs): def main(): - file1_path = '../GPS/gpsLnew.csv' - file2_path = '../GPS/gpsLold2.cvs' - file3_path = '../GPS/gpsLold.csv' + file1_path = '../Data/GPS/gpsLnew.csv' + file2_path = '../Data/GPS/gpsLold2.cvs' + file3_path = '../Data/GPS/gpsLold.csv' common_pairs, new_pairs, old_pairs, data1, data2 = compare_csv(file1_path, file2_path) diff --git a/src/compareS.py b/first_try/compareS.py similarity index 96% rename from src/compareS.py rename to first_try/compareS.py index 89083c8..1818ea9 100644 --- a/src/compareS.py +++ b/first_try/compareS.py @@ -109,7 +109,7 @@ def create_import_list(path, path_new, old_pairs, new_pairs, common_pairs): # print(import_df) # pd.reset_option('display.max_rows') # pd.reset_option('display.max_columns') - out_path = '../GPS/outputSchueler.csv' + out_path = '../Data/GPS/outputSchueler.csv' import_df.to_csv(out_path, sep=';', index=False) print(len(import_df)) print('Schüler Import Liste erzeugt') @@ -117,9 +117,9 @@ def create_import_list(path, path_new, old_pairs, new_pairs, common_pairs): def main(): - file1_path = '../GPS/gpsSnew.csv' - file2_path = '../GPS/gpsSold2.cvs' - file3_path = '../GPS/gpsSold.csv' + file1_path = '../Data/GPS/gpsSnew.csv' + file2_path = '../Data/GPS/gpsSold2.cvs' + file3_path = '../Data/GPS/gpsSold.csv' common_pairs, new_pairs, old_pairs, data1, data2 = compare_csv(file1_path, file2_path) diff --git a/src/format_csv.py b/first_try/format_csv.py similarity index 100% rename from src/format_csv.py rename to first_try/format_csv.py diff --git a/first_try/main.py b/first_try/main.py new file mode 100644 index 0000000..f47035e --- /dev/null +++ b/first_try/main.py @@ -0,0 +1,26 @@ +from first_try.format_csv import * + +if __name__ == "__main__": + oldlist = 'Data/alte-liste-utf.csv' + newlist = 'Data/neue-liste-utf.csv' + + test = 'GPS/lehrer_HL0707113.csv_intern' + new_test = 'GPS/gpsLold.csv' + clean = 'GPS/gpsLold2.cvs' + + # Liste nach Fehler prüfen und Zeichen ersetzen + check_file(test) + format_csv(test, new_test) + clean_data(new_test, clean) + + # path_old_csv = input('Pfad zur alten Liste eingeben eingeben: ') + # format_csv(path_old_csv, oldlist) + # path_new_csv = input('Pfad zur neuen Liste eingeben eingeben: ') + # format_csv(path_new_csv, newlist) + + # Generierte Listen mit pandas öffnen + # Spaltennamen prüfen und überflüssige Spalten löschen + # Abgleich starten -collisions + # TODO Testuser + Spezialfälle Raumaccounts etc. "Tafel" "Raum" + # TODO Fehlerhafte Sonderzeichen erkennen (oxport_schueler.csv - 171) + # TODO File / Ausgabe von Gesamten Zahlen zum Abgleich mit DryRun (ImportLines, Created, Modified, Delete) diff --git a/src/output.py b/first_try/output.py similarity index 100% rename from src/output.py rename to first_try/output.py diff --git a/main.py b/main.py index c43e983..0435b14 100644 --- a/main.py +++ b/main.py @@ -1,25 +1,61 @@ -from src.format_csv import * +######################################## +# -- Python Script für UCS Import -- # +# -- by Patrick vom Hagen 2024 -- # +######################################## + + +############################################################### +# IMPORT - standard Python imports für benötigte Bibliotheken # + +import pandas as pd # pandas für Datenmanagement +import chardet # chardet erkennt Formatierung - Umwandlung des erkannten Formats in UTF-8 +import csv # zur Verarbeitung von .csv Dateien +import uuid # zur Generierung von neuen UUIDs + +############################# +# Flags / globale Variablen # + +del_zeros = False # Boolean, ob führende Nullen bei Klassen entfernt werden sollen +get_typos = False # Boolean, ob geringe Unterschiede zwischen den Listen ausgegeben werden sollen +test_user = [] # Array in dem Spezialuser gespeichert und wieder eingefügt werden + +school_id = 'HL' # HL-Tag für Klassen +ox_context = 0 # OX Context pro Schule +mail_quota_lul = 2048 # MailUserQuota LuL +ox_quota_lul = 20480 # oxUserQuota LuL +mail_quota_sus = 1024 # MailUserQuota SuS +ox_quota_sus = 5120 # oxUserQuota LuL + +####################### +# MAIN FUNCTION START # if __name__ == "__main__": - oldlist = 'Data/alte-liste-utf.csv' - newlist = 'Data/neue-liste-utf.csv' + print("Schul-IT UCS-Import Tool:") - test = 'GPS/lehrer_HL0707113.csv_intern' - new_test = 'GPS/gpsLold.csv' - clean = 'GPS/gpsLold2.cvs' + # ----------------------------------------------------------------- # + # Step 1 - Dateien wählen, formatieren, einlesen | Variablen setzen # - # Liste nach Fehler prüfen und Zeichen ersetzen - check_file(test) - format_csv(test, new_test) - clean_data(new_test, clean) + # ToDo Daten mit UI einlesen + lehrer_liste_neu = "./Data/test_new.csv" + lehrer_liste_system = "./Data/test.csv" - # path_old_csv = input('Pfad zur alten Liste eingeben eingeben: ') - # format_csv(path_old_csv, oldlist) - # path_new_csv = input('Pfad zur neuen Liste eingeben eingeben: ') - # format_csv(path_new_csv, newlist) + schueler_liste_neu = "./Data/test_new.csv" + schueler_liste_system = "./Data/test.csv" - # Generierte Listen mit pandas öffnen - # Spaltennamen prüfen und überflüssige Spalten löschen - # Abgleich starten -collisions - # TODO Testuser + Spezialfälle Raumaccounts etc. "Tafel" "Raum" - # TODO Fehlerhafte Sonderzeichen erkennen (oxport_schueler.csv - 171) + # Variablen füllen: + school_id = input("Schul-ID eingeben (Format: 'HL070XXXX):") + ox_context = input("OX-Context der Schule eingeben:") + if input("Klassen mit führenden Nullen? (Y/N) ").lower() == "y": + del_zeros = False + else: + del_zeros = True + + # ---------------------------------------------------- # + # Step 2 - auf name, vorname reduzieren und abgleichen # + + # 2.1 Data Frames für Abgleich erstellen + # 2.2 Datei mit gesamten Anzahlen generieren und füllen + # 2.3 Nicht zuordnungsbare bzw. neue Einträge nach Abweichung zur Bestandsliste prüfen und ausgeben + # UUIDs zwischenspeichern? + + # Step 3 - Import Data generieren - klasse, uuids, weiteres in einer Liste zusammenführen diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/restart.py b/src/restart.py deleted file mode 100644 index dc5895f..0000000 --- a/src/restart.py +++ /dev/null @@ -1,22 +0,0 @@ -######################################## -# -- Python Script für UCS Import -- # -# -- by Patrick vom Hagen 2024 -- # -######################################## - -# IMPORT - standard Python imports für benötigte Bibliotheken # - -import pandas as pd # pandas für Datenmanagement -import chardet # chardet erkennt Formatierung - Umwandlung des erkannten Formats in UTF-8 -import csv # zur Verarbeitung von .csv Dateien -import uuid # zur Generierung von neuen UUIDs - -# Flags / globale Variablen # - -del_zeros = False # Boolean, ob führende Nullen bei Klassen entfernt werden sollen -get_typos = False # Boolean, ob geringe Unterschiede zwischen den Listen ausgegeben werden sollen - -ox_context = 0 # OX Context pro Schule -mail_quota_lul = 2048 # MailUserQuota LuL -ox_quota_lul = 20480 # oxUserQuota LuL -mail_quota_sus = 1024 # MailUserQuota SuS -ox_quota_sus = 5120 # oxUserQuota LuL diff --git a/src/step1.py b/src/step1.py new file mode 100644 index 0000000..e69de29