diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..88e9278 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,20 @@ +// For format details, see https://aka.ms/devcontainer.json. +{ + "name": "Ledger SQLite", + "context": "..", + "dockerFile": "../Dockerfile", + "containerEnv": { + "DEBIAN_FRONTEND": "noninteractive", + "LEDGER_FILE": "/workspaces/ledger-sqlite/data/example.ledger", + "OUTPUT_FILE": "/workspaces/ledger-sqlite/data/example.sqlite" + }, + "onCreateCommand": "/usr/local/bin/python -m pip install -U mypy pylint black", + "customizations": { + "vscode": { + "extensions": [ + "ms-python.python", + "ms-python.vscode-pylance" + ] + } + } +} \ No newline at end of file diff --git a/.gitignore b/.gitignore index b6e4761..1afa7af 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +data/* +!data/example.ledger + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] @@ -55,31 +58,6 @@ coverage.xml *.mo *.pot -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py # pyenv .python-version @@ -94,13 +72,6 @@ ipython_config.py # PEP 582; used by e.g. github.com/David-OConnor/pyflow __pypackages__/ -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - # Environments .env .venv @@ -110,16 +81,9 @@ ENV/ env.bak/ venv.bak/ -# Spyder project settings -.spyderproject -.spyproject - # Rope project settings .ropeproject -# mkdocs documentation -/site - # mypy .mypy_cache/ .dmypy.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..69e556c --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,18 @@ +{ + "python.formatting.provider": "black", + "python.linting.enabled": true, + "python.linting.pylintEnabled": true, + "python.linting.mypyEnabled": true, + "python.linting.mypyCategorySeverity.error":"Warning", + "python.linting.mypyArgs": [ + "--follow-imports=silent", + "--show-column-numbers", + "--disallow-untyped-defs", + "--disallow-incomplete-defs", + "--disallow-untyped-calls", + "--warn-redundant-casts", + "--warn-return-any", + "--strict", + "--strict-equality" + ] +} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..11d92a6 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,15 @@ +# TODO: Change this to a multiarch hledger image +FROM dastapov/hledger:1.26 as hledger + +FROM library/python:3.10-slim + +COPY --from=hledger /usr/bin/hledger /usr/bin/hledger + +COPY src /code + +# TODO: Support more than one ledger file +ENV LEDGER_FILE=/data/all.ledger +ENV OUTPUT_FILE=/data/all.sqlite +ENV OVERWRITE_OUTPUT=true + +ENTRYPOINT ["python", "/code/export.py"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..1ac2d69 --- /dev/null +++ b/README.md @@ -0,0 +1,11 @@ +# Ledger SQLite + +A simple containerized utility which creates a SQLite database from the contents of a ledger file. Uses [hledger](https://hledger.org) to output JSON which is then transformed into a database schema. + +Exports are not designed to be consistent across iterations. It is recommended that you start from a fresh databse every export. + +## Usage + +TBD + +Currently planning on designing the container to be able to run as a long-lived job with a cron schedule for exporting. There will also be a method for running as a one-off. diff --git a/data/example.ledger b/data/example.ledger new file mode 100644 index 0000000..c460f21 --- /dev/null +++ b/data/example.ledger @@ -0,0 +1,53 @@ +; A sample journal file. +; +; Sets up this account tree: +; assets +; bank +; checking +; saving +; cash +; expenses +; food +; supplies +; income +; gifts +; salary +; liabilities +; debts + +; declare accounts: +; account assets:bank:checking +; account income:salary +; account income:gifts +; account assets:bank:saving +; account assets:cash +; account expenses:food +; account expenses:supplies +; account liabilities:debts + +; declare commodities: +; commodity $ + +2008/01/01 income + assets:bank:checking $1 + income:salary + +2008/06/01 gift + assets:bank:checking $1 + income:gifts + +2008/06/02 save + assets:bank:saving $1 + assets:bank:checking + +2008/06/03 * eat & shop + expenses:food $1 + expenses:supplies $1 + assets:cash + +2008/12/31 * pay off + liabilities:debts $1 + assets:bank:checking + + +;final comment \ No newline at end of file diff --git a/src/db.py b/src/db.py new file mode 100644 index 0000000..5507fb6 --- /dev/null +++ b/src/db.py @@ -0,0 +1,59 @@ +import sqlite3 +from typing import List + +from ledger import Transaction + + +class CursorContextManager: + """ + CursorContextManager is a ContextManager for a SQLite database. Provides a + Cursor to interact with the database. + """ + + _filename: str + _conn: sqlite3.Connection + _cursor: sqlite3.Cursor + + def __init__(self, filename: str): + self._filename = filename + + def __enter__(self): + self._conn = sqlite3.connect(self._filename) + self._cursor = self._conn.cursor() + return self._cursor + + def __exit__(self, exc_type, exc_value, exc_tb): + self._conn.commit() + self._cursor.close() + self._conn.close() + + +def create_tables(cursor: sqlite3.Cursor) -> None: + """ + Create the necessary tables in the provided database. + """ + + # SQLite data types: https://www.sqlite.org/datatype3.html + cursor.execute( + """ + CREATE TABLE transactions + ( + date text, + check_num text, + note text, + account text, + amount real, + cleared integer, + tags text + ) + """ + ) + + +def write_transactions(cursor: sqlite3.Cursor, txs: List[Transaction]) -> None: + """ + Convert the transactions into a format that can be stored in the database + and then store it. + """ + + print(txs[0]) diff --git a/src/ledger.py b/src/ledger.py new file mode 100644 index 0000000..614c693 --- /dev/null +++ b/src/ledger.py @@ -0,0 +1,72 @@ +import json +import subprocess +from typing import List +import typing + + +class AmountQuantity(typing.TypedDict): + decimalMantissa: int + decimalPlaces: int + floatingPoint: int + + +class AmountStyle(typing.TypedDict): + ascommodityside: str + ascommodityspaced: bool + asdecimalpoint: str + asdigitgroups: str | None # TODO: str? + asprecision: int + + +class Amount(typing.TypedDict): + acommodity: str + aprice: str | None # TODO: str or int? + aquantity: AmountQuantity + astyle: AmountStyle + + +class Posting(typing.TypedDict): + paccount: str + pamount: List[Amount] + pbalanceassertion: str | None + pcomment: str + pdate: str | None + pdate2: str | None + poriginal: str | None + pstatus: str # TODO: Enum? + ptags: List[str] + ptransaction_: int + ptype: str # TODO: Enum? + + +class SourcePosition(typing.TypedDict): + sourceColumn: int + sourceLine: int + sourceName: str + + +class Transaction(typing.TypedDict): + tcode: str + tcomment: str + tdate: str + tdate2: str | None + tdescription: str + tindex: int + tpostings: List[Posting] + tprecedingcomment: str + tsourcepos: List[SourcePosition] + tstatus: str # TODO: Enum? + ttags: List[str] + + +def get_transactions(filename: str) -> List[Transaction]: + """ + Retrieve a list of transaction objects as parsed by hledger from the provided input file. + """ + res = subprocess.run( + ["/usr/bin/hledger", "print", "-f", filename, "-O", "json"], + check=True, + capture_output=True, + ) + txs: List[Transaction] = json.loads(res.stdout) + return txs diff --git a/src/main.py b/src/main.py new file mode 100644 index 0000000..0a758eb --- /dev/null +++ b/src/main.py @@ -0,0 +1,39 @@ +import os +import os.path +import sys + +from db import CursorContextManager, create_tables, write_transactions +from ledger import get_transactions + + +# TODO: Replace with os.environ.get() and produce nice error messages. +ledger_file = os.environ["LEDGER_FILE"] +output_file = os.environ["OUTPUT_FILE"] +overwrite_output = os.environ["OVERWRITE_OUTPUT"] + +print(f"Converting {ledger_file} to {output_file}") + + +def to_bool(val: str) -> bool: + """ + Return if the provided string is consider truthy. + """ + return val.lower() in ["true", "1", "t", "y", "yes"] + + +if os.path.exists(output_file): + if not to_bool(overwrite_output): + print("Output file exists. Set OVERWRITE_OUTPUT=true to overwrite.") + sys.exit(1) + else: + print("Output file exists. Overwriting.") + os.remove(output_file) + +txs = get_transactions(ledger_file) + +with CursorContextManager(filename=output_file) as db: + create_tables(db) + write_transactions(db, txs) + + +print("Conversion complete.")