yutsuo
/
robopato
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

updates

master
Yutsuo 2 years ago
parent
a143898a1b
commit
f993402f5b
4 changed files with 4287 additions and 1122 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 98
      nb_account_etl.ipynb
  2. 2203
      nb_credit_card_etl.ipynb
  3. 3090
      nb_lab.ipynb
  4. 12
      queries.sql

98
nb_account_etl.ipynb
Unescape View File

@ -2,43 +2,16 @@
"cells": [ "cells": [
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 2, "execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
"# import csv\n",
"\n",
"# with open(\"./documents/extrato.csv\", newline=\"\", encoding=\"latin\") as csvfile:\n",
"# spamreader = csv.reader(csvfile)\n",
"# for row in spamreader:\n",
"# print(\", \".join(row))"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"# import csv\n", "insert_query = \"INSERT IGNORE INTO default.TRANSACTION (ID, TDATE, ACCOUNT_ID, MEMO, COUNTRY, OUTFLOW, INFLOW, OWNER_ID, INSTALLMENT_NR, INSTALLMENT_TT, BILL, CREATED, UPDATED) VALUES ( %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s )\""
"\n",
"# with open(\"./documents/extrato.csv\", newline=\"\", encoding=\"latin\") as csvfile:\n",
"# spamreader = csv.reader(csvfile)\n",
"# csv_list = list(spamreader)\n",
"\n",
"# print(csv_list[1:])\n",
"# print(csv_list[1:][0])\n",
"# print(csv_list[1:][0][0])\n",
"# print(csv_list[1:][0][0].split(sep=\"/\"))\n",
"# print(csv_list[1:][0][0].split(sep=\"/\")[0])\n",
"\n",
"# for item in csv_list[1:]:\n",
"# print(item)"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 9, "execution_count": 11,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -78,6 +51,7 @@
" else:\n", " else:\n",
" tInstallmentNr = None\n", " tInstallmentNr = None\n",
" tInstallmentTt = None\n", " tInstallmentTt = None\n",
" tBill = \"CC\"\n",
" tCreated = str(datetime.now(tz=None))\n", " tCreated = str(datetime.now(tz=None))\n",
" tUpdated = None\n", " tUpdated = None\n",
"\n", "\n",
@ -98,26 +72,28 @@
" tOwner,\n", " tOwner,\n",
" tInstallmentNr,\n", " tInstallmentNr,\n",
" tInstallmentTt,\n", " tInstallmentTt,\n",
" tBill,\n",
" tCreated,\n", " tCreated,\n",
" tUpdated,\n", " tUpdated,\n",
" )\n", " )\n",
" )\n", " )\n",
"\n", " print(\"DONE\")\n",
" return insert_bulk" " return insert_bulk"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 10, "execution_count": 12,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"insert_query = \"INSERT IGNORE INTO default.TRANSACTION (ID, TDATE, ACCOUNT_ID, MEMO, COUNTRY, OUTFLOW, INFLOW, OWNER_ID, INSTALLMENT_NR, INSTALLMENT_TT, CREATED, UPDATED) VALUES ( %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s )\"" "# Test\n",
"# build_account_input(\"./documents/extrato-30d.csv\")"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 11, "execution_count": 13,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -144,7 +120,17 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 12, "execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
"# Test\n",
"# db_insert(build_account_input(\"./documents/extrato-30d.csv\"))"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -164,14 +150,52 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 13, "execution_count": 16,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
"name": "stdout", "name": "stdout",
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"executor(): Error\n", "DONE\n",
"CONNECTED! <mysql.connector.connection_cext.CMySQLConnection object at 0x000001A7BC1ADDF0>\n",
"DONE!\n",
"DONE\n",
"CONNECTED! <mysql.connector.connection_cext.CMySQLConnection object at 0x000001A7BC1ADDF0>\n",
"DONE!\n",
"DONE\n",
"CONNECTED! <mysql.connector.connection_cext.CMySQLConnection object at 0x000001A7BC1D61E0>\n",
"DONE!\n",
"DONE\n",
"CONNECTED! <mysql.connector.connection_cext.CMySQLConnection object at 0x000001A7BC1D77D0>\n",
"DONE!\n",
"DONE\n",
"CONNECTED! <mysql.connector.connection_cext.CMySQLConnection object at 0x000001A7BAD26C00>\n",
"DONE!\n",
"DONE\n",
"CONNECTED! <mysql.connector.connection_cext.CMySQLConnection object at 0x000001A7BC1D5820>\n",
"DONE!\n",
"DONE\n",
"CONNECTED! <mysql.connector.connection_cext.CMySQLConnection object at 0x000001A7BC1D5820>\n",
"DONE!\n",
"DONE\n",
"CONNECTED! <mysql.connector.connection_cext.CMySQLConnection object at 0x000001A7BC1708F0>\n",
"DONE!\n",
"DONE\n",
"CONNECTED! <mysql.connector.connection_cext.CMySQLConnection object at 0x000001A7BC1D5820>\n",
"DONE!\n",
"DONE\n",
"CONNECTED! <mysql.connector.connection_cext.CMySQLConnection object at 0x000001A7BC137B00>\n",
"DONE!\n",
"DONE\n",
"CONNECTED! <mysql.connector.connection_cext.CMySQLConnection object at 0x000001A7BC208200>\n",
"DONE!\n",
"DONE\n",
"CONNECTED! <mysql.connector.connection_cext.CMySQLConnection object at 0x000001A7BC23FFB0>\n",
"DONE!\n",
"DONE\n",
"CONNECTED! <mysql.connector.connection_cext.CMySQLConnection object at 0x000001A7BC1D5880>\n",
"DONE!\n",
"EXECUTOR COMPLETED.\n" "EXECUTOR COMPLETED.\n"
] ]
} }

2203
nb_credit_card_etl.ipynb
View File

File diff suppressed because one or more lines are too long

3090
nb_lab.ipynb
View File

File diff suppressed because it is too large Load Diff

12
queries.sql
Unescape View File

@ -5,3 +5,15 @@ TRUNCATE TABLE `default`.`TRANSACTION`;
UPDATE `TRANSACTION` SET MEMO = TRIM(MEMO) UPDATE `TRANSACTION` SET MEMO = TRIM(MEMO)
ALTER TABLE `default`.SUBCATEGORY MODIFY COLUMN UPDATED datetime ON UPDATE CURRENT_TIMESTAMP NULL; ALTER TABLE `default`.SUBCATEGORY MODIFY COLUMN UPDATED datetime ON UPDATE CURRENT_TIMESTAMP NULL;
SELECT * FROM `default`.`TRANSACTION` t WHERE INSTALLMENT_TT IS NOT NULL;
SELECT * FROM `default`.`TRANSACTION` t WHERE t.ID NOT IN default.CATEGORIZED_TRANSACTIONS.ID;
SELECT *
FROM TRANSACTION t
LEFT JOIN CATEGORIZED_TRANSACTIONS ct
ON t.ID = ct.TRANSACTION_ID
WHERE ct.TRANSACTION_ID IS NULL;
SELECT * FROM default.TRANSACTION t LEFT JOIN default.CATEGORIZED_TRANSACTIONS ct ON t.ID = ct.TRANSACTION_ID WHERE ct.TRANSACTION_ID IS NULL;
Write Preview
Loading…
Cancel
Save