diff --git a/db/data_transfer.R b/db/data_transfer.R new file mode 100644 index 0000000..97bc4eb --- /dev/null +++ b/db/data_transfer.R @@ -0,0 +1,421 @@ +## +## FileMaker -> SQLite (canonical english schema) +## Keeps Hibiscus payload fields unchanged +## + +library(tidyverse) +library(dbx) +library(RJDBC) +library(DBI) +library(RSQLite) + +source("~/R/rfunc/fehler_add.R") + +ok <- TRUE +error_f <- NULL + +# Settings ---- + +fm_host <- "localhost" +fm_port <- 2399 +fm_db <- "gemfin04" +fm_user <- "admin" +fm_pass <- "" +fm_driver_class <- "com.filemaker.jdbc.Driver" +fm_driver_jar <- "~/R/fmjdbc.jar" + +sqlite_path <- "db/development.sqlite" + +# optional future switch +USE_CENTS <- FALSE + +# Helpers ---- + +to_cents <- function(x) { + if (is.null(x)) return(NA_integer_) + if (is.factor(x)) x <- as.character(x) + + if (is.character(x)) { + x <- trimws(x) + x[x == ""] <- NA + x <- gsub("\\.", "", x) + x <- gsub(",", ".", x) + } + + d <- suppressWarnings(as.numeric(x)) + ifelse(is.na(d), NA_integer_, as.integer(round(d * 100))) +} + +exec_sql <- function(sql) dbExecute(con_s, sql) + +# Connections ---- + +if (ok) { + drv <- JDBC(fm_driver_class, fm_driver_jar) + con_f <- dbConnect( + drv, + paste0("jdbc:filemaker://", fm_host, ":", fm_port, "/", fm_db), + fm_user, + fm_pass + ) + ok <- !con_f@jc$isClosed() + error_f <- fehler_add("FileMaker connection established", ok, error_f) +} + +if (ok) { + if (file.exists(sqlite_path)) file.remove(sqlite_path) + con_s <- dbConnect(SQLite(), sqlite_path, extended_types = TRUE) + dbExecute(con_s, "PRAGMA foreign_keys = ON;") + error_f <- fehler_add("SQLite connection established", TRUE, error_f) +} + +# Schema creation ---- + +if (ok) { + + exec_sql("CREATE TABLE accounts ( + id INTEGER PRIMARY KEY, + account_name TEXT NOT NULL, + bank_name TEXT, + hibiscus_account_id INTEGER, + budget_id INTEGER, + wiso_account TEXT, + bank_account_no TEXT, + is_donations INTEGER, + notes TEXT, + created_at TEXT, + updated_at TEXT + );") + + exec_sql("CREATE TABLE budgets ( + id INTEGER PRIMARY KEY, + row_no INTEGER, + area TEXT, + label TEXT, + direction TEXT, + created_at TEXT, + updated_at TEXT + );") + + exec_sql("CREATE TABLE contacts ( + id INTEGER PRIMARY KEY, + first_name TEXT, + last_name TEXT, + street TEXT, + postal_code TEXT, + city TEXT, + country TEXT, + phone TEXT, + mobile TEXT, + email TEXT, + salutation TEXT, + gender TEXT, + donor TEXT, + member INTEGER, + notes TEXT, + receipt_salutation TEXT, + receipt_name TEXT, + display_name TEXT, + partner_id INTEGER, + is_couple INTEGER, + is_company INTEGER, + created_at TEXT, + updated_at TEXT + );") + + exec_sql("CREATE TABLE bank_connections ( + id INTEGER PRIMARY KEY, + contact_id INTEGER, + contact_text TEXT, + iban TEXT, + bic TEXT, + bank_name TEXT, + remote_name TEXT, + created_at TEXT, + updated_at TEXT + );") + + exec_sql("CREATE TABLE entries ( + id INTEGER PRIMARY KEY, + contact_id INTEGER, + purpose TEXT, + note TEXT, + remote_name TEXT, + created_at TEXT, + updated_at TEXT + );") + + if (!USE_CENTS) { + exec_sql("CREATE TABLE postings ( + id INTEGER PRIMARY KEY, + entry_id INTEGER NOT NULL, + account_id INTEGER NOT NULL, + valuta TEXT NOT NULL, + booking_date TEXT, + amount NUMERIC NOT NULL, + invoice_no TEXT, + note TEXT, + status TEXT, + waived INTEGER, + bank_transaction_id INTEGER, + wiso_id INTEGER, + receipt_id INTEGER, + project_id INTEGER, + coin_share_amount NUMERIC, + created_at TEXT, + updated_at TEXT + );") + } else { + exec_sql("CREATE TABLE postings ( + id INTEGER PRIMARY KEY, + entry_id INTEGER NOT NULL, + account_id INTEGER NOT NULL, + valuta TEXT NOT NULL, + booking_date TEXT, + amount_cents INTEGER NOT NULL, + invoice_no TEXT, + note TEXT, + status TEXT, + waived INTEGER, + bank_transaction_id INTEGER, + wiso_id INTEGER, + receipt_id INTEGER, + project_id INTEGER, + coin_share_cents INTEGER, + created_at TEXT, + updated_at TEXT + );") + } + + # Hibiscus payload unchanged ---- + exec_sql("CREATE TABLE hibiscus_transactions ( + id INTEGER PRIMARY KEY, + konto_id INTEGER, + empfaenger_konto TEXT, + empfaenger_blz TEXT, + empfaenger_name TEXT, + betrag REAL, + zweck TEXT, + zweck2 TEXT, + zweck3 TEXT, + datum TEXT, + valuta TEXT, + saldo REAL, + primanota INTEGER, + art TEXT, + customerref TEXT, + kommentar TEXT, + checksum REAL, + umsatztyp_id INTEGER, + flags REAL, + gvcode INTEGER, + addkey INTEGER, + txid TEXT, + purposecode TEXT, + endtoendid TEXT, + mandateid TEXT, + empfaenger_name2 TEXT, + creditorid TEXT + );") + + error_f <- fehler_add("Schema created", TRUE, error_f) +} + +# Transfer Accounts ---- + +if (ok) { + accounts <- dbxSelect(con_f, " + SELECT id, konto, budget_id, bankname, updated_at, created_at, + konto_hibiscus, konto_wiso, kontonummer_bank, notizen, b_spenden + FROM Konten + ") %>% + transmute( + id = as.integer(id), + account_name = konto, + bank_name = bankname, + hibiscus_account_id = as.integer(konto_hibiscus), + budget_id = as.integer(budget_id), + wiso_account = konto_wiso, + bank_account_no = kontonummer_bank, + is_donations = as.integer(b_spenden), + notes = notizen, + created_at = created_at, + updated_at = updated_at + ) + + dbWriteTable(con_s, "accounts", accounts, append = TRUE) +} + +if (ok) { + projects <- dbxSelect(con_f, " + SELECT id, projektname, bereich, notiz, jahr, created_at, updated_at from Projekte") %>% + transmute( + id = as.integer(id), + projektname = projektname, + bereich = bereich, + notiz = notiz, + jahr = as.integer(jahr), + created_at = created_at, + updated_at = updated_at + ) + + dbWriteTable(con_s, "projects", projects, append = TRUE) +} + +# Transfer Budgets ---- + +if (ok) { + budgets <- dbxSelect(con_f, " + SELECT id, richtung, bereich, bezeichnung, reihe + FROM Budgets + ") %>% + transmute( + id = as.integer(id), + row_no = as.integer(reihe), + area = bereich, + label = bezeichnung, + direction = richtung, + created_at = format(Sys.time(), ""), + updated_at = format(Sys.time(), "") + ) + + dbWriteTable(con_s, "budgets", budgets, append = TRUE) +} + +# Transfer Contacts + +if (ok) { + contacts <- dbxSelect(con_f, " + SELECT id, vorname, nachname, strasse, plz, ort, Land, + telefon, mobil, email, anrede, geschlecht, spender, mitglied, notiz, + quittung_anrede, quittung_name, bezeichnung, partner_id, b_paar, b_firma + FROM Adressen + ") %>% + as_tibble() %>% + transmute( + id = as.integer(id), + first_name = vorname, + last_name = nachname, + street = strasse, + postal_code = plz, + city = ort, + country = Land, + phone = telefon, + mobile = mobil, + email = email, + salutation = anrede, + gender = geschlecht, + donor = spender, + member = as.integer(mitglied), + notes = notiz, + receipt_salutation = quittung_anrede, + receipt_name = quittung_name, + display_name = bezeichnung, + partner_id = as.integer(partner_id), + is_couple = as.integer(b_paar), + is_company = as.integer(b_firma), + created_at = format(Sys.time(), ""), + updated_at = format(Sys.time(), "") + ) + + dbWriteTable(con_s, "contacts", contacts, append = TRUE) +} + +# Transfer Entries ---- + +if (ok) { + entries <- dbxSelect(con_f, " + SELECT id, adress_id, verwendungszweck, notiz, kontakt, created_at, updated_at + FROM trans + ") %>% + as_tibble() %>% + transmute( + id = as.integer(id), + contact_id = as.integer(adress_id), + purpose = verwendungszweck, + note = notiz, + remote_name = kontakt, + created_at = created_at, + updated_at = updated_at + ) + + dbWriteTable(con_s, "entries", entries, append = TRUE) +} + +# Transfer Postings ---- + +if (ok) { + postings <- dbxSelect(con_f, " + SELECT id, trans_id, konto_id, wertstellung, buchungsdatum, betrag, + rechnungsnummer, notiz, status, b_verzicht, umsatz_id, wiso_id, + quittung_id, projekt_id, betrag_muenzen, created_at, updated_at + FROM buchungen + ") %>% + as_tibble() + + if (!USE_CENTS) { + postings <- postings %>% + transmute( + id = as.integer(id), + entry_id = as.integer(trans_id), + account_id = as.integer(konto_id), + valuta = as.character(wertstellung), + booking_date = as.character(buchungsdatum), + amount = betrag, + invoice_no = rechnungsnummer, + note = notiz, + status = status, + waived = as.integer(b_verzicht), + bank_transaction_id = as.integer(umsatz_id), + wiso_id = as.integer(wiso_id), + receipt_id = as.integer(quittung_id), + project_id = as.integer(projekt_id), + coin_share_amount = betrag_muenzen, + created_at = created_at, + updated_at = updated_at + ) + } else { + postings <- postings %>% + transmute( + id = as.integer(id), + entry_id = as.integer(trans_id), + account_id = as.integer(konto_id), + valuta = as.character(wertstellung), + booking_date = as.character(buchungsdatum), + amount_cents = to_cents(betrag), + invoice_no = rechnungsnummer, + note = notiz, + status = status, + waived = as.integer(b_verzicht), + bank_transaction_id = as.integer(umsatz_id), + wiso_id = as.integer(wiso_id), + receipt_id = as.integer(quittung_id), + project_id = as.integer(projekt_id), + coin_share_cents = to_cents(betrag_muenzen), + created_at = created_at, + updated_at = updated_at + ) + } + + dbWriteTable(con_s, "postings", postings, append = TRUE) +} + +# Transfer Hibiscus transactions ---- + +if (ok) { + spalten <- dbxSelect(con_f, "SELECT * FROM Umsatz WHERE id=1") %>% + names %>% + paste(collapse = ",") + query <- paste("SELECT", spalten, "FROM Umsatz") + hib <- dbxSelect(con_f, query) %>% + select(-fz_jahr) + dbWriteTable(con_s, "hibiscus_transactions", hib, append = TRUE) +} + +# Close ---- + +dbDisconnect(con_s) +dbDisconnect(con_f) + +print("Migration finished.") + diff --git a/db/development.sqlite3 b/db/development.sqlite3 index 65a0603..f7eecda 100644 Binary files a/db/development.sqlite3 and b/db/development.sqlite3 differ