Work in progress parsing output from hledger

.devcontainer/devcontainer.json

@@ -0,0 +1,20 @@
+// For format details, see https://aka.ms/devcontainer.json.
+{
+	"name": "Ledger SQLite",
+	"context": "..",
+	"dockerFile": "../Dockerfile",
+	"containerEnv": {
+		"DEBIAN_FRONTEND": "noninteractive",
+		"LEDGER_FILE": "/workspaces/ledger-sqlite/data/example.ledger",
+		"OUTPUT_FILE": "/workspaces/ledger-sqlite/data/example.sqlite"
+	},
+	"onCreateCommand": "/usr/local/bin/python -m pip install -U mypy pylint black",
+	"customizations": {
+		"vscode": {
+			"extensions": [
+				"ms-python.python",
+				"ms-python.vscode-pylance"
+			]
+		}
+	}
+}

.gitignore

@@ -1,3 +1,6 @@
+data/*
+!data/example.ledger
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -55,31 +58,6 @@ coverage.xml
 *.mo
 *.pot
 
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-db.sqlite3-journal
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# IPython
-profile_default/
-ipython_config.py
 
 # pyenv
 .python-version
@@ -94,13 +72,6 @@ ipython_config.py
 # PEP 582; used by e.g. github.com/David-OConnor/pyflow
 __pypackages__/
 
-# Celery stuff
-celerybeat-schedule
-celerybeat.pid
-
-# SageMath parsed files
-*.sage.py
-
 # Environments
 .env
 .venv
@@ -110,16 +81,9 @@ ENV/
 env.bak/
 venv.bak/
 
-# Spyder project settings
-.spyderproject
-.spyproject
-
 # Rope project settings
 .ropeproject
 
-# mkdocs documentation
-/site
-
 # mypy
 .mypy_cache/
 .dmypy.json

.vscode/settings.json

@@ -0,0 +1,18 @@
+{
+    "python.formatting.provider": "black",
+    "python.linting.enabled": true,
+    "python.linting.pylintEnabled": true,
+    "python.linting.mypyEnabled": true,
+    "python.linting.mypyCategorySeverity.error":"Warning",
+    "python.linting.mypyArgs": [
+        "--follow-imports=silent",
+        "--show-column-numbers",
+        "--disallow-untyped-defs",
+        "--disallow-incomplete-defs",
+        "--disallow-untyped-calls",
+        "--warn-redundant-casts",
+        "--warn-return-any",
+        "--strict",
+        "--strict-equality"
+    ]
+}

Dockerfile

@@ -0,0 +1,15 @@
+# TODO: Change this to a multiarch hledger image
+FROM dastapov/hledger:1.26 as hledger
+
+FROM library/python:3.10-slim
+
+COPY --from=hledger /usr/bin/hledger /usr/bin/hledger
+
+COPY src /code
+
+# TODO: Support more than one ledger file
+ENV LEDGER_FILE=/data/all.ledger
+ENV OUTPUT_FILE=/data/all.sqlite
+ENV OVERWRITE_OUTPUT=true
+
+ENTRYPOINT ["python", "/code/export.py"]

README.md

@@ -0,0 +1,11 @@
+# Ledger SQLite
+
+A simple containerized utility which creates a SQLite database from the contents of a ledger file. Uses [hledger](https://hledger.org) to output JSON which is then transformed into a database schema.
+
+Exports are not designed to be consistent across iterations. It is recommended that you start from a fresh databse every export.
+
+## Usage
+
+TBD
+
+Currently planning on designing the container to be able to run as a long-lived job with a cron schedule for exporting. There will also be a method for running as a one-off.

data/example.ledger

@@ -0,0 +1,53 @@
+; A sample journal file.
+;
+; Sets up this account tree:
+; assets
+;   bank
+;     checking
+;     saving
+;   cash
+; expenses
+;   food
+;   supplies
+; income
+;   gifts
+;   salary
+; liabilities
+;   debts
+
+; declare accounts:
+; account assets:bank:checking
+; account income:salary
+; account income:gifts
+; account assets:bank:saving
+; account assets:cash
+; account expenses:food
+; account expenses:supplies
+; account liabilities:debts
+
+; declare commodities:
+; commodity $
+
+2008/01/01 income
+    assets:bank:checking  $1
+    income:salary
+
+2008/06/01 gift
+    assets:bank:checking  $1
+    income:gifts
+
+2008/06/02 save
+    assets:bank:saving  $1
+    assets:bank:checking
+
+2008/06/03 * eat & shop
+    expenses:food      $1
+    expenses:supplies  $1
+    assets:cash
+
+2008/12/31 * pay off
+    liabilities:debts  $1
+    assets:bank:checking
+
+
+;final comment

src/db.py

@@ -0,0 +1,59 @@
+import sqlite3
+from typing import List
+
+from ledger import Transaction
+
+
+class CursorContextManager:
+    """
+    CursorContextManager is a ContextManager for a SQLite database. Provides a
+    Cursor to interact with the database.
+    """
+
+    _filename: str
+    _conn: sqlite3.Connection
+    _cursor: sqlite3.Cursor
+
+    def __init__(self, filename: str):
+        self._filename = filename
+
+    def __enter__(self):
+        self._conn = sqlite3.connect(self._filename)
+        self._cursor = self._conn.cursor()
+        return self._cursor
+
+    def __exit__(self, exc_type, exc_value, exc_tb):
+        self._conn.commit()
+        self._cursor.close()
+        self._conn.close()
+
+
+def create_tables(cursor: sqlite3.Cursor) -> None:
+    """
+    Create the necessary tables in the provided database.
+    """
+
+    # SQLite data types: https://www.sqlite.org/datatype3.html
+    cursor.execute(
+        """
+        CREATE TABLE transactions
+        (
+            date text,
+            check_num text,
+            note text,
+            account text,
+            amount real,
+            cleared integer,
+            tags text
+        )
+        """
+    )
+
+
+def write_transactions(cursor: sqlite3.Cursor, txs: List[Transaction]) -> None:
+    """
+    Convert the transactions into a format that can be stored in the database
+    and then store it.
+    """
+
+    print(txs[0])

src/ledger.py

@@ -0,0 +1,72 @@
+import json
+import subprocess
+from typing import List
+import typing
+
+
+class AmountQuantity(typing.TypedDict):
+    decimalMantissa: int
+    decimalPlaces: int
+    floatingPoint: int
+
+
+class AmountStyle(typing.TypedDict):
+    ascommodityside: str
+    ascommodityspaced: bool
+    asdecimalpoint: str
+    asdigitgroups: str | None  # TODO: str?
+    asprecision: int
+
+
+class Amount(typing.TypedDict):
+    acommodity: str
+    aprice: str | None  # TODO: str or int?
+    aquantity: AmountQuantity
+    astyle: AmountStyle
+
+
+class Posting(typing.TypedDict):
+    paccount: str
+    pamount: List[Amount]
+    pbalanceassertion: str | None
+    pcomment: str
+    pdate: str | None
+    pdate2: str | None
+    poriginal: str | None
+    pstatus: str  # TODO: Enum?
+    ptags: List[str]
+    ptransaction_: int
+    ptype: str  # TODO: Enum?
+
+
+class SourcePosition(typing.TypedDict):
+    sourceColumn: int
+    sourceLine: int
+    sourceName: str
+
+
+class Transaction(typing.TypedDict):
+    tcode: str
+    tcomment: str
+    tdate: str
+    tdate2: str | None
+    tdescription: str
+    tindex: int
+    tpostings: List[Posting]
+    tprecedingcomment: str
+    tsourcepos: List[SourcePosition]
+    tstatus: str  # TODO: Enum?
+    ttags: List[str]
+
+
+def get_transactions(filename: str) -> List[Transaction]:
+    """
+    Retrieve a list of transaction objects as parsed by hledger from the provided input file.
+    """
+    res = subprocess.run(
+        ["/usr/bin/hledger", "print", "-f", filename, "-O", "json"],
+        check=True,
+        capture_output=True,
+    )
+    txs: List[Transaction] = json.loads(res.stdout)
+    return txs

src/main.py

@@ -0,0 +1,39 @@
+import os
+import os.path
+import sys
+
+from db import CursorContextManager, create_tables, write_transactions
+from ledger import get_transactions
+
+
+# TODO: Replace with os.environ.get() and produce nice error messages.
+ledger_file = os.environ["LEDGER_FILE"]
+output_file = os.environ["OUTPUT_FILE"]
+overwrite_output = os.environ["OVERWRITE_OUTPUT"]
+
+print(f"Converting {ledger_file} to {output_file}")
+
+
+def to_bool(val: str) -> bool:
+    """
+    Return if the provided string is consider truthy.
+    """
+    return val.lower() in ["true", "1", "t", "y", "yes"]
+
+
+if os.path.exists(output_file):
+    if not to_bool(overwrite_output):
+        print("Output file exists. Set OVERWRITE_OUTPUT=true to overwrite.")
+        sys.exit(1)
+    else:
+        print("Output file exists. Overwriting.")
+        os.remove(output_file)
+
+txs = get_transactions(ledger_file)
+
+with CursorContextManager(filename=output_file) as db:
+    create_tables(db)
+    write_transactions(db, txs)
+
+
+print("Conversion complete.")