diff --git a/db/data_transfer.R b/db/data_transfer.R index 0f310c3..0c38e66 100755 --- a/db/data_transfer.R +++ b/db/data_transfer.R @@ -26,30 +26,14 @@ fm_driver_jar <- "~/R/fmjdbc.jar" sqlite_path <- "db/development.sqlite" -# optional future switch -USE_CENTS <- FALSE # Helpers ---- - -to_cents <- function(x) { - if (is.null(x)) return(NA_integer_) - if (is.factor(x)) x <- as.character(x) - - if (is.character(x)) { - x <- trimws(x) - x[x == ""] <- NA - x <- gsub("\\.", "", x) - x <- gsub(",", ".", x) - } - - d <- suppressWarnings(as.numeric(x)) - ifelse(is.na(d), NA_integer_, as.integer(round(d * 100))) -} - exec_sql <- function(sql) dbExecute(con_s, sql) +f_anzahl <- function(tabelle){ +dbxSelect(con_s, paste0("SELECt count(id) FROM ", tabelle)) %>% pull +} # Connections ---- - if (ok) { drv <- JDBC(fm_driver_class, fm_driver_jar) con_f <- dbConnect( @@ -69,201 +53,61 @@ if (ok) { error_f <- fehler_add("SQLite connection established", TRUE, error_f) } -# Schema creation ---- - +## * Transfer Accounts ---- if (ok) { - - exec_sql("CREATE TABLE accounts ( + exec_sql("DROP TABLE IF EXISTS konten;") + exec_sql("CREATE TABLE konten ( id INTEGER PRIMARY KEY, - account_name TEXT NOT NULL, + kontoname TEXT NOT NULL, bank_name TEXT, - hibiscus_account_id INTEGER, + hibiscus_konto_id INTEGER, budget_id INTEGER, - wiso_account TEXT, - bank_account_no TEXT, - is_donations INTEGER, - notes TEXT, + wiso_kategorie TEXT, + bank_konto_nr TEXT, + ist_spende INTEGER, + notiz TEXT, created_at TEXT, updated_at TEXT );") - exec_sql("CREATE TABLE budgets ( - id INTEGER PRIMARY KEY, - row_no INTEGER, - area TEXT, - label TEXT, - direction TEXT, - created_at TEXT, - updated_at TEXT - );") - - exec_sql("CREATE TABLE contacts ( - id INTEGER PRIMARY KEY, - first_name TEXT, - last_name TEXT, - street TEXT, - postal_code TEXT, - city TEXT, - country TEXT, - phone TEXT, - mobile TEXT, - email TEXT, - salutation TEXT, - gender TEXT, - donor TEXT, - member INTEGER, - notes TEXT, - receipt_salutation TEXT, - receipt_name TEXT, - display_name TEXT, - partner_id INTEGER, - is_couple INTEGER, - is_company INTEGER, - created_at TEXT, - updated_at TEXT - );") - - exec_sql("CREATE TABLE bank_connections ( - id INTEGER PRIMARY KEY, - contact_id INTEGER, - contact_text TEXT, - iban TEXT, - bic TEXT, - bank_name TEXT, - remote_name TEXT, - created_at TEXT, - updated_at TEXT - );") - - exec_sql("CREATE TABLE entries ( - id INTEGER PRIMARY KEY, - contact_id INTEGER, - purpose TEXT, - note TEXT, - remote_name TEXT, - created_at TEXT, - updated_at TEXT - );") - - if (!USE_CENTS) { - exec_sql("CREATE TABLE postings ( - id INTEGER PRIMARY KEY, - entry_id INTEGER NOT NULL, - account_id INTEGER NOT NULL, - valuta TEXT NOT NULL, - booking_date TEXT, - amount NUMERIC NOT NULL, - invoice_no TEXT, - note TEXT, - status TEXT, - waived INTEGER, - bank_transaction_id INTEGER, - wiso_id INTEGER, - receipt_id INTEGER, - project_id INTEGER, - coin_share_amount NUMERIC, - created_at TEXT, - updated_at TEXT - );") - } else { - exec_sql("CREATE TABLE postings ( - id INTEGER PRIMARY KEY, - entry_id INTEGER NOT NULL, - account_id INTEGER NOT NULL, - valuta TEXT NOT NULL, - booking_date TEXT, - amount_cents INTEGER NOT NULL, - invoice_no TEXT, - note TEXT, - status TEXT, - waived INTEGER, - bank_transaction_id INTEGER, - wiso_id INTEGER, - receipt_id INTEGER, - project_id INTEGER, - coin_share_cents INTEGER, - created_at TEXT, - updated_at TEXT - );") - } - - # Hibiscus payload unchanged ---- - exec_sql("CREATE TABLE hibiscus_transactions ( - id INTEGER PRIMARY KEY, - konto_id INTEGER, - empfaenger_konto TEXT, - empfaenger_blz TEXT, - empfaenger_name TEXT, - betrag REAL, - zweck TEXT, - zweck2 TEXT, - zweck3 TEXT, - datum TEXT, - valuta TEXT, - saldo REAL, - primanota INTEGER, - art TEXT, - customerref TEXT, - kommentar TEXT, - checksum REAL, - umsatztyp_id INTEGER, - flags REAL, - gvcode INTEGER, - addkey INTEGER, - txid TEXT, - purposecode TEXT, - endtoendid TEXT, - mandateid TEXT, - empfaenger_name2 TEXT, - creditorid TEXT - );") - - # Attachments ---- - exec_sql("CREATE TABLE attachments ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - entry_id INTEGER, - quittung_id INTEGER, - adress_id INTEGER, - wiso_id INTEGER, - btisch_id TEXT, - original_name TEXT, - kategorie TEXT, - path TEXT NOT NULL, - note TEXT, - created_at TEXT DEFAULT (datetime('now')), - updated_at TEXT DEFAULT (datetime('now')) -);") - - error_f <- fehler_add("Schema created", TRUE, error_f) -} - -# Transfer Accounts ---- - -if (ok) { - accounts <- dbxSelect(con_f, " + konten <- dbxSelect(con_f, " SELECT id, konto, budget_id, bankname, updated_at, created_at, konto_hibiscus, konto_wiso, kontonummer_bank, notizen, ist_spende FROM Konten ") %>% transmute( id = as.integer(id), - account_name = konto, + kontoname = konto, bank_name = bankname, - hibiscus_account_id = as.integer(konto_hibiscus), + hibiscus_konto_id = as.integer(konto_hibiscus), budget_id = as.integer(budget_id), - wiso_account = konto_wiso, - bank_account_no = kontonummer_bank, - is_donations = as.integer(ist_spende), - notes = notizen, + wiso_kategorie = konto_wiso, + bank_konto_nr = kontonummer_bank, + ist_spende = as.integer(ist_spende), + notiz = notizen, created_at = created_at, updated_at = updated_at ) - - dbWriteTable(con_s, "accounts", accounts, append = TRUE) + dbWriteTable(con_s, "konten", konten, append = TRUE) + ok <- dbExistsTable(con_s, "konten") + ok <- f_anzahl("konten") == nrow(konten) & ok + error_f <- fehler_add("Konten übertragen", TRUE, error_f) } +## * Transfer Projekte ---- if (ok) { - projects <- dbxSelect(con_f, " + exec_sql("DROP TABLE IF EXISTS projekte;") + exec_sql("CREATE TABLE projekte ( + id INTEGER PRIMARY KEY, + projektname TEXT NOT NULL, + jahr INTEGER, + projekt_wiso TEXT, + bereich TEXT, + notiz TEXT, + created_at TEXT, + updated_at TEXT + );") + projekte <- dbxSelect(con_f, " SELECT id, projektname, bereich, notiz, jahr, created_at, updated_at from Projekte") %>% transmute( id = as.integer(id), @@ -275,152 +119,247 @@ if (ok) { updated_at = updated_at ) - dbWriteTable(con_s, "projects", projects, append = TRUE) + dbWriteTable(con_s, "projekte", projekte, append = TRUE) + ok <- dbExistsTable(con_s, "projekte") + ok <- f_anzahl("projekte") == nrow(projekte) & ok + error_f <- fehler_add("Projekte übertragen", TRUE, error_f) } -# Transfer Budgets ---- - +## * Transfer Budgets ---- if (ok) { + exec_sql("DROP TABLE IF EXISTS budgets;") + exec_sql("CREATE TABLE budgets ( + id INTEGER PRIMARY KEY, + reihe INTEGER, + bereich TEXT, + bezeichnung TEXT, + richtung TEXT, + created_at TEXT, + updated_at TEXT + );") budgets <- dbxSelect(con_f, " SELECT id, richtung, bereich, bezeichnung, reihe FROM Budgets ") %>% transmute( id = as.integer(id), - row_no = as.integer(reihe), - area = bereich, - label = bezeichnung, - direction = richtung, + reihe = as.integer(reihe), + bereich = bereich, + bezeichnung = bezeichnung, + richtung = richtung, created_at = format(Sys.time(), ""), updated_at = format(Sys.time(), "") ) - dbWriteTable(con_s, "budgets", budgets, append = TRUE) + ok <- dbExistsTable(con_s, "budgets") + ok <- f_anzahl("budgets") == nrow(budgets) & ok + error_f <- fehler_add("Budgets übertragen", TRUE, error_f) } -# Transfer Contacts +### ** Transfer Budget-Werte + if(ok){ + exec_sql("DROP TABLE IF EXISTS budget_werte;") + exec_sql("CREATE TABLE budget_werte ( + id INTEGER PRIMARY KEY, + budget_id INTEGER, + jahr INTEGER, + betrag DECIMAL, + budget_listen_nr INTEGER, + created_at TEXT, + updated_at TEXT + );") + budget_werte <- dbReadTable(con_f, "Budgets_werte") %>% + transmute( + id = as.integer(id), + budget_id = as.integer(budget_id), + jahr = as.integer(jahr), + betrag = betrag, + budget_listen_nr = as.integer(budget_listen_nr), + created_at = format(Sys.time(), ""), + updated_at = format(Sys.time(), "") + ) + dbWriteTable(con_s, "budget_werte", budget_werte, append = TRUE) + ok <- dbExistsTable(con_s, "budget_werte") + ok <- f_anzahl("budget_werte") == nrow(budget_werte) & ok + error_f <- fehler_add("budget_werte übertragen", TRUE, error_f) +} ## ------------------------------------------------------- 2026-06-19 08:52 + +## * Transfer Adressen ---- if (ok) { - contacts <- dbxSelect(con_f, " + exec_sql("DROP TABLE IF EXISTS adressen;") + exec_sql("CREATE TABLE adressen ( + id INTEGER PRIMARY KEY, + vorname TEXT, + nachname TEXT, + strasse TEXT, + plz TEXT, + ort TEXT, + land TEXT, + telefon TEXT, + mobil TEXT, + email TEXT, + anrede TEXT, + geschlecht TEXT, + ist_mitglied INTEGER, + ist_firma INTEGER, + quittung_anrede TEXT, + quittung_name TEXT, + display_name TEXT, + partner_id INTEGER, + notiz TEXT, + created_at TEXT, + updated_at TEXT + );") + + adressen <- dbxSelect(con_f, " SELECT id, vorname, nachname, strasse, plz, ort, Land, - telefon, mobil, email, anrede, geschlecht, spender, mitglied, notiz, + telefon, mobil, email, anrede, geschlecht, mitglied, notiz, quittung_anrede, quittung_name, bezeichnung, partner_id, b_paar, b_firma FROM Adressen ") %>% as_tibble() %>% transmute( id = as.integer(id), - first_name = vorname, - last_name = nachname, - street = strasse, - postal_code = plz, - city = ort, - country = Land, - phone = telefon, - mobile = mobil, + vorname = vorname, + nachname = nachname, + strasse = strasse, + plz = plz, + ort = ort, + land = Land, + telefon = telefon, + mobil = mobil, email = email, - salutation = anrede, - gender = geschlecht, - donor = spender, - member = as.integer(mitglied), - notes = notiz, - receipt_salutation = quittung_anrede, - receipt_name = quittung_name, + anrede = anrede, + geschlecht = geschlecht, + ist_mitglied = as.integer(mitglied), + ist_firma = as.integer(b_firma), + quittung_anrede = quittung_anrede, + quittung_name = quittung_name, display_name = bezeichnung, partner_id = as.integer(partner_id), - is_couple = as.integer(b_paar), - is_company = as.integer(b_firma), + notiz = notiz, created_at = format(Sys.time(), ""), updated_at = format(Sys.time(), "") ) - - dbWriteTable(con_s, "contacts", contacts, append = TRUE) + dbWriteTable(con_s, "adressen", adressen, append = TRUE) + ok <- dbExistsTable(con_s, "adressen") + ok <- f_anzahl("adressen") == nrow(adressen) & ok + error_f <- fehler_add("adressen übertragen", TRUE, error_f) } -# Transfer Entries ---- +## * Transfer Transaktionen ---- if (ok) { - entries <- dbxSelect(con_f, " + exec_sql("DROP TABLE IF EXISTS transaktionen;") + exec_sql("CREATE TABLE transaktionen ( + id INTEGER PRIMARY KEY, + adress_id INTEGER, + vwz TEXT, + notiz TEXT, + bank_kontakt TEXT, + created_at TEXT, + updated_at TEXT + );") + + transaktionen <- dbxSelect(con_f, " SELECT id, adress_id, verwendungszweck, notiz, kontakt, created_at, updated_at FROM trans ") %>% as_tibble() %>% transmute( id = as.integer(id), - contact_id = as.integer(adress_id), - purpose = verwendungszweck, - note = notiz, - remote_name = kontakt, + adress_id = as.integer(adress_id), + vwz = verwendungszweck, + notiz = notiz, + bank_kontakt = kontakt, created_at = created_at, updated_at = updated_at ) - dbWriteTable(con_s, "entries", entries, append = TRUE) + dbWriteTable(con_s, "transaktionen", transaktionen, append = TRUE) + ok <- dbExistsTable(con_s, "transaktionen") + ok <- f_anzahl("transaktionen") == nrow(transaktionen) & ok + error_f <- fehler_add("transaktionen übertragen", TRUE, error_f) } -# Transfer Postings ---- +## * Transfer Postings ---- if (ok) { - postings <- dbxSelect(con_f, " + exec_sql("DROP TABLE IF EXISTS buchungen;") + exec_sql("CREATE TABLE buchungen ( + id INTEGER PRIMARY KEY, + trans_id INTEGER NOT NULL, + konto_id INTEGER NOT NULL, + valuta TEXT NOT NULL, + buchungsdatum TEXT, + betrag DECIMAL, + rechnungsnummer TEXT, + notiz TEXT, + status TEXT, + ist_verzicht INTEGER, + umsatz_id INTEGER, + wiso_id INTEGER, + quittung_id INTEGER, + projekt_id INTEGER, + betrag_muenzen DECIMAL, + created_at TEXT, + updated_at TEXT + );") + buchungen <- dbxSelect(con_f, " SELECT id, trans_id, konto_id, wertstellung, buchungsdatum, betrag, rechnungsnummer, notiz, status, b_verzicht, umsatz_id, wiso_id, quittung_id, projekt_id, betrag_muenzen, created_at, updated_at FROM buchungen ") %>% as_tibble() - - if (!USE_CENTS) { - postings <- postings %>% + buchungen <- buchungen %>% transmute( id = as.integer(id), - entry_id = as.integer(trans_id), - account_id = as.integer(konto_id), + trans_id = as.integer(trans_id), + konto_id = as.integer(konto_id), valuta = as.character(wertstellung), - booking_date = as.character(buchungsdatum), - amount = betrag, - invoice_no = rechnungsnummer, - note = notiz, + buchungsdatum = as.character(buchungsdatum), + betrag = betrag, + rechnungsnummer = rechnungsnummer, + notiz = notiz, status = status, - waived = as.integer(b_verzicht), - bank_transaction_id = as.integer(umsatz_id), + ist_verzicht = as.integer(b_verzicht), + umsatz_id = as.integer(umsatz_id), wiso_id = as.integer(wiso_id), - receipt_id = as.integer(quittung_id), - project_id = as.integer(projekt_id), - coin_share_amount = betrag_muenzen, + quittung_id = as.integer(quittung_id), + projekt_id = as.integer(projekt_id), + betrag_muenzen = betrag_muenzen, created_at = created_at, updated_at = updated_at ) - } else { - postings <- postings %>% - transmute( - id = as.integer(id), - entry_id = as.integer(trans_id), - account_id = as.integer(konto_id), - valuta = as.character(wertstellung), - booking_date = as.character(buchungsdatum), - amount_cents = to_cents(betrag), - invoice_no = rechnungsnummer, - note = notiz, - status = status, - waived = as.integer(b_verzicht), - bank_transaction_id = as.integer(umsatz_id), - wiso_id = as.integer(wiso_id), - receipt_id = as.integer(quittung_id), - project_id = as.integer(projekt_id), - coin_share_cents = to_cents(betrag_muenzen), - created_at = created_at, - updated_at = updated_at - ) - } - - dbWriteTable(con_s, "postings", postings, append = TRUE) + dbWriteTable(con_s, "buchungen", buchungen, append = TRUE) + ok <- dbExistsTable(con_s, "buchungen") + ok <- f_anzahl("buchungen") == nrow(buchungen) & ok + error_f <- fehler_add("buchungen übertragen", TRUE, error_f) } -# Daten aus Attachments Tabelle übertragen ---- +## * Transfer Attachments ---- if(ok){ + exec_sql("DROP TABLE IF EXISTS attachments;") + exec_sql("CREATE TABLE attachments ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + trans_id INTEGER, + quittung_id INTEGER, + adress_id INTEGER, + wiso_id INTEGER, + btisch_id TEXT, + original_name TEXT, + kategorie TEXT, + ext TEXT NOT NULL, + notiz TEXT, + created_at TEXT DEFAULT (datetime('now')), + updated_at TEXT DEFAULT (datetime('now')) +);") + att <- dbxSelect(con_f, "SELECT id, trans_id, quittung_id, wiso_id, btisch_id, - adress_id, ft_dateiname, beschreibung, created_at, updated_at, ft_extension FROM Attachments") %>% - mutate( + adress_id, ft_dateiname, beschreibung, created_at, updated_at, ft_extension FROM Dokumente") %>% + transmute( id = as.integer(id), trans_id = as.integer(trans_id), quittung_id = as.integer(quittung_id), @@ -429,19 +368,18 @@ if(ok){ adress_id = as.integer(adress_id), created_at = as.character(created_at), updated_at = as.character(updated_at), - path = paste0(id,".", ft_extension) - ) %>% - rename( - entry_id = trans_id, - note = beschreibung, - original_name = ft_dateiname - ) %>% - select(-ft_extension) - dbWriteTable(con_s, "Attachments", att, append = T) + original_name = ft_dateiname, + ext = paste0(id,".", ft_extension) + ) + dbWriteTable(con_s, "attachments", att, append = T) + ok <- dbExistsTable(con_s, "attachments") + ok <- f_anzahl("attachments") == nrow(att) & ok + error_f <- fehler_add("attachments übertragen", TRUE, error_f) + } ## ------------------------------------------------------- 2026-03-19 16:38 -## * Daeien übertragen ---- +### ** Dateien übertragen ---- if(ok){ pfad <- "~/Insync/Projekte/Gemeindefinanzen/gemfin-fm/gemfin04/Dokumente/datei/" zielpfad <- "~/Documents/workspace/gemfin-shiny/www/documents/" @@ -465,60 +403,94 @@ if(ok){ } ## ------------------------------------------------------- 2026-04-28 18:40 -# Transfer Bankverbindungen ---- +## * Transfer Bankverbindungen ---- if (ok) { - bank_connections <- dbxSelect(con_f, " + exec_sql("DROP TABLE IF EXISTS bvb;") + exec_sql("CREATE TABLE bvb ( + id INTEGER PRIMARY KEY, + adress_id INTEGER, + remote_name TEXT, + iban TEXT, + bic TEXT, + kreditinstitut TEXT, + created_at TEXT, + updated_at TEXT + );") + bvb <- dbxSelect(con_f, " SELECT id, adress_id, kontakt, iban, bic, kreditinstitut, - remote_name, created_at, updated_at + created_at, updated_at FROM Bankverbindungen ") %>% transmute( id = as.integer(id), - contact_id = as.integer(adress_id), - contact_text = kontakt, + adress_id = as.integer(adress_id), + remote_name = kontakt, iban = iban, bic = bic, - bank_name = kreditinstitut, - remote_name = remote_name, + kreditinstitut = kreditinstitut, created_at = as.character(created_at), updated_at = as.character(updated_at) ) - - sum(duplicated(bank_connections$id)) - dbWriteTable(con_s, "bank_connections", bank_connections, append = TRUE) - error_f <- fehler_add( - paste(nrow(bank_connections), "Bankverbindungen übertragen"), TRUE, error_f - ) + dbWriteTable(con_s, "bvb", bvb, append = TRUE) + ok <- dbExistsTable(con_s, "bvb") + ok <- f_anzahl("bvb") == nrow(bvb) & ok + error_f <- fehler_add("bvb übertragen", TRUE, error_f) } + +## * Transfer Hibiscus ---- + if(ok){ + exec_sql("DROP TABLE IF EXISTS umsatz;") + exec_sql("CREATE TABLE umsatz ( + id INTEGER PRIMARY KEY, + konto_id INTEGER, + empfaenger_konto TEXT, + empfaenger_blz TEXT, + empfaenger_name TEXT, + betrag REAL, + zweck TEXT, + zweck2 TEXT, + zweck3 TEXT, + datum TEXT, + valuta TEXT, + saldo REAL, + customerref TEXT, + kommentar TEXT, + endtoendid TEXT + );") + umsatz <- dbReadTable(con_f, "Umsatz") %>% + transmute( + id = as.integer(id), + konto_id = as.integer(konto_id), + empfaenger_konto = empfaenger_konto, + empfaenger_blz = empfaenger_blz, + empfaenger_name = empfaenger_name, + betrag = betrag, + zweck = zweck, + zweck2 = zweck2, + zweck3 = zweck2, + datum = datum, + valuta = valuta, + saldo = saldo, + kommentar = kommentar, + endtoendid = endtoendid + ) + dbWriteTable(con_s, "umsatz", umsatz, append = TRUE) + ok <- sum(duplicated(umsatz$id)) == 0 + ok <- dbExistsTable(con_s, "umsatz") & ok + ok <- f_anzahl("umsatz") == nrow(umsatz) & ok + error_f <- fehler_add("Tabelle Umsatz existiert", ok, error_f) +} ## ------------------------------------------------------- 2026-06-19 08:34 + + # Close ---- +dbListTables(con_s) dbDisconnect(con_s) dbDisconnect(con_f) print("Migration finished.") -## * Dateien umbenennen ---- -library("tools") -atts <- dbReadTable(conn, "attachments") -sum(duplicated(atts$id)) == 0 -pfad <- "www/attachments/" -for( ind in 1:nrow(atts)){ - datei <- atts$original_name[ind] - if( file.exists(paste0(pfad, datei)) ){ - # print(ind) - ext <- file_ext(datei) - idwert <- atts$id[ind] - if(is.na(idwert)) break - name_neu <- paste0(idwert,".",ext) - print(name_neu) - file.copy(from = paste0(pfad, datei), to=paste0("www/documents/", name_neu), overwrite = T) - } -} -atts %>% - filter(id == 1379) - -which( "1379.pdf" %in% atts$original_name) diff --git a/db/development.sqlite b/db/development.sqlite index 75446ae..7fe9976 100755 Binary files a/db/development.sqlite and b/db/development.sqlite differ