Work in progress parsing output from hledger
This commit is contained in:
parent
a893d56681
commit
ca78ecd9f0
20
.devcontainer/devcontainer.json
Normal file
20
.devcontainer/devcontainer.json
Normal file
@ -0,0 +1,20 @@
|
||||
// For format details, see https://aka.ms/devcontainer.json.
|
||||
{
|
||||
"name": "Ledger SQLite",
|
||||
"context": "..",
|
||||
"dockerFile": "../Dockerfile",
|
||||
"containerEnv": {
|
||||
"DEBIAN_FRONTEND": "noninteractive",
|
||||
"LEDGER_FILE": "/workspaces/ledger-sqlite/data/example.ledger",
|
||||
"OUTPUT_FILE": "/workspaces/ledger-sqlite/data/example.sqlite"
|
||||
},
|
||||
"onCreateCommand": "/usr/local/bin/python -m pip install -U mypy pylint black",
|
||||
"customizations": {
|
||||
"vscode": {
|
||||
"extensions": [
|
||||
"ms-python.python",
|
||||
"ms-python.vscode-pylance"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
42
.gitignore
vendored
42
.gitignore
vendored
@ -1,3 +1,6 @@
|
||||
data/*
|
||||
!data/example.ledger
|
||||
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
@ -55,31 +58,6 @@ coverage.xml
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
local_settings.py
|
||||
db.sqlite3
|
||||
db.sqlite3-journal
|
||||
|
||||
# Flask stuff:
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# Scrapy stuff:
|
||||
.scrapy
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
|
||||
# PyBuilder
|
||||
target/
|
||||
|
||||
# Jupyter Notebook
|
||||
.ipynb_checkpoints
|
||||
|
||||
# IPython
|
||||
profile_default/
|
||||
ipython_config.py
|
||||
|
||||
# pyenv
|
||||
.python-version
|
||||
@ -94,13 +72,6 @@ ipython_config.py
|
||||
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
|
||||
__pypackages__/
|
||||
|
||||
# Celery stuff
|
||||
celerybeat-schedule
|
||||
celerybeat.pid
|
||||
|
||||
# SageMath parsed files
|
||||
*.sage.py
|
||||
|
||||
# Environments
|
||||
.env
|
||||
.venv
|
||||
@ -110,16 +81,9 @@ ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
.spyproject
|
||||
|
||||
# Rope project settings
|
||||
.ropeproject
|
||||
|
||||
# mkdocs documentation
|
||||
/site
|
||||
|
||||
# mypy
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
|
18
.vscode/settings.json
vendored
Normal file
18
.vscode/settings.json
vendored
Normal file
@ -0,0 +1,18 @@
|
||||
{
|
||||
"python.formatting.provider": "black",
|
||||
"python.linting.enabled": true,
|
||||
"python.linting.pylintEnabled": true,
|
||||
"python.linting.mypyEnabled": true,
|
||||
"python.linting.mypyCategorySeverity.error":"Warning",
|
||||
"python.linting.mypyArgs": [
|
||||
"--follow-imports=silent",
|
||||
"--show-column-numbers",
|
||||
"--disallow-untyped-defs",
|
||||
"--disallow-incomplete-defs",
|
||||
"--disallow-untyped-calls",
|
||||
"--warn-redundant-casts",
|
||||
"--warn-return-any",
|
||||
"--strict",
|
||||
"--strict-equality"
|
||||
]
|
||||
}
|
15
Dockerfile
Normal file
15
Dockerfile
Normal file
@ -0,0 +1,15 @@
|
||||
# TODO: Change this to a multiarch hledger image
|
||||
FROM dastapov/hledger:1.26 as hledger
|
||||
|
||||
FROM library/python:3.10-slim
|
||||
|
||||
COPY --from=hledger /usr/bin/hledger /usr/bin/hledger
|
||||
|
||||
COPY src /code
|
||||
|
||||
# TODO: Support more than one ledger file
|
||||
ENV LEDGER_FILE=/data/all.ledger
|
||||
ENV OUTPUT_FILE=/data/all.sqlite
|
||||
ENV OVERWRITE_OUTPUT=true
|
||||
|
||||
ENTRYPOINT ["python", "/code/export.py"]
|
11
README.md
Normal file
11
README.md
Normal file
@ -0,0 +1,11 @@
|
||||
# Ledger SQLite
|
||||
|
||||
A simple containerized utility which creates a SQLite database from the contents of a ledger file. Uses [hledger](https://hledger.org) to output JSON which is then transformed into a database schema.
|
||||
|
||||
Exports are not designed to be consistent across iterations. It is recommended that you start from a fresh databse every export.
|
||||
|
||||
## Usage
|
||||
|
||||
TBD
|
||||
|
||||
Currently planning on designing the container to be able to run as a long-lived job with a cron schedule for exporting. There will also be a method for running as a one-off.
|
53
data/example.ledger
Normal file
53
data/example.ledger
Normal file
@ -0,0 +1,53 @@
|
||||
; A sample journal file.
|
||||
;
|
||||
; Sets up this account tree:
|
||||
; assets
|
||||
; bank
|
||||
; checking
|
||||
; saving
|
||||
; cash
|
||||
; expenses
|
||||
; food
|
||||
; supplies
|
||||
; income
|
||||
; gifts
|
||||
; salary
|
||||
; liabilities
|
||||
; debts
|
||||
|
||||
; declare accounts:
|
||||
; account assets:bank:checking
|
||||
; account income:salary
|
||||
; account income:gifts
|
||||
; account assets:bank:saving
|
||||
; account assets:cash
|
||||
; account expenses:food
|
||||
; account expenses:supplies
|
||||
; account liabilities:debts
|
||||
|
||||
; declare commodities:
|
||||
; commodity $
|
||||
|
||||
2008/01/01 income
|
||||
assets:bank:checking $1
|
||||
income:salary
|
||||
|
||||
2008/06/01 gift
|
||||
assets:bank:checking $1
|
||||
income:gifts
|
||||
|
||||
2008/06/02 save
|
||||
assets:bank:saving $1
|
||||
assets:bank:checking
|
||||
|
||||
2008/06/03 * eat & shop
|
||||
expenses:food $1
|
||||
expenses:supplies $1
|
||||
assets:cash
|
||||
|
||||
2008/12/31 * pay off
|
||||
liabilities:debts $1
|
||||
assets:bank:checking
|
||||
|
||||
|
||||
;final comment
|
59
src/db.py
Normal file
59
src/db.py
Normal file
@ -0,0 +1,59 @@
|
||||
import sqlite3
|
||||
from typing import List
|
||||
|
||||
from ledger import Transaction
|
||||
|
||||
|
||||
class CursorContextManager:
|
||||
"""
|
||||
CursorContextManager is a ContextManager for a SQLite database. Provides a
|
||||
Cursor to interact with the database.
|
||||
"""
|
||||
|
||||
_filename: str
|
||||
_conn: sqlite3.Connection
|
||||
_cursor: sqlite3.Cursor
|
||||
|
||||
def __init__(self, filename: str):
|
||||
self._filename = filename
|
||||
|
||||
def __enter__(self):
|
||||
self._conn = sqlite3.connect(self._filename)
|
||||
self._cursor = self._conn.cursor()
|
||||
return self._cursor
|
||||
|
||||
def __exit__(self, exc_type, exc_value, exc_tb):
|
||||
self._conn.commit()
|
||||
self._cursor.close()
|
||||
self._conn.close()
|
||||
|
||||
|
||||
def create_tables(cursor: sqlite3.Cursor) -> None:
|
||||
"""
|
||||
Create the necessary tables in the provided database.
|
||||
"""
|
||||
|
||||
# SQLite data types: https://www.sqlite.org/datatype3.html
|
||||
cursor.execute(
|
||||
"""
|
||||
CREATE TABLE transactions
|
||||
(
|
||||
date text,
|
||||
check_num text,
|
||||
note text,
|
||||
account text,
|
||||
amount real,
|
||||
cleared integer,
|
||||
tags text
|
||||
)
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def write_transactions(cursor: sqlite3.Cursor, txs: List[Transaction]) -> None:
|
||||
"""
|
||||
Convert the transactions into a format that can be stored in the database
|
||||
and then store it.
|
||||
"""
|
||||
|
||||
print(txs[0])
|
72
src/ledger.py
Normal file
72
src/ledger.py
Normal file
@ -0,0 +1,72 @@
|
||||
import json
|
||||
import subprocess
|
||||
from typing import List
|
||||
import typing
|
||||
|
||||
|
||||
class AmountQuantity(typing.TypedDict):
|
||||
decimalMantissa: int
|
||||
decimalPlaces: int
|
||||
floatingPoint: int
|
||||
|
||||
|
||||
class AmountStyle(typing.TypedDict):
|
||||
ascommodityside: str
|
||||
ascommodityspaced: bool
|
||||
asdecimalpoint: str
|
||||
asdigitgroups: str | None # TODO: str?
|
||||
asprecision: int
|
||||
|
||||
|
||||
class Amount(typing.TypedDict):
|
||||
acommodity: str
|
||||
aprice: str | None # TODO: str or int?
|
||||
aquantity: AmountQuantity
|
||||
astyle: AmountStyle
|
||||
|
||||
|
||||
class Posting(typing.TypedDict):
|
||||
paccount: str
|
||||
pamount: List[Amount]
|
||||
pbalanceassertion: str | None
|
||||
pcomment: str
|
||||
pdate: str | None
|
||||
pdate2: str | None
|
||||
poriginal: str | None
|
||||
pstatus: str # TODO: Enum?
|
||||
ptags: List[str]
|
||||
ptransaction_: int
|
||||
ptype: str # TODO: Enum?
|
||||
|
||||
|
||||
class SourcePosition(typing.TypedDict):
|
||||
sourceColumn: int
|
||||
sourceLine: int
|
||||
sourceName: str
|
||||
|
||||
|
||||
class Transaction(typing.TypedDict):
|
||||
tcode: str
|
||||
tcomment: str
|
||||
tdate: str
|
||||
tdate2: str | None
|
||||
tdescription: str
|
||||
tindex: int
|
||||
tpostings: List[Posting]
|
||||
tprecedingcomment: str
|
||||
tsourcepos: List[SourcePosition]
|
||||
tstatus: str # TODO: Enum?
|
||||
ttags: List[str]
|
||||
|
||||
|
||||
def get_transactions(filename: str) -> List[Transaction]:
|
||||
"""
|
||||
Retrieve a list of transaction objects as parsed by hledger from the provided input file.
|
||||
"""
|
||||
res = subprocess.run(
|
||||
["/usr/bin/hledger", "print", "-f", filename, "-O", "json"],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
)
|
||||
txs: List[Transaction] = json.loads(res.stdout)
|
||||
return txs
|
39
src/main.py
Normal file
39
src/main.py
Normal file
@ -0,0 +1,39 @@
|
||||
import os
|
||||
import os.path
|
||||
import sys
|
||||
|
||||
from db import CursorContextManager, create_tables, write_transactions
|
||||
from ledger import get_transactions
|
||||
|
||||
|
||||
# TODO: Replace with os.environ.get() and produce nice error messages.
|
||||
ledger_file = os.environ["LEDGER_FILE"]
|
||||
output_file = os.environ["OUTPUT_FILE"]
|
||||
overwrite_output = os.environ["OVERWRITE_OUTPUT"]
|
||||
|
||||
print(f"Converting {ledger_file} to {output_file}")
|
||||
|
||||
|
||||
def to_bool(val: str) -> bool:
|
||||
"""
|
||||
Return if the provided string is consider truthy.
|
||||
"""
|
||||
return val.lower() in ["true", "1", "t", "y", "yes"]
|
||||
|
||||
|
||||
if os.path.exists(output_file):
|
||||
if not to_bool(overwrite_output):
|
||||
print("Output file exists. Set OVERWRITE_OUTPUT=true to overwrite.")
|
||||
sys.exit(1)
|
||||
else:
|
||||
print("Output file exists. Overwriting.")
|
||||
os.remove(output_file)
|
||||
|
||||
txs = get_transactions(ledger_file)
|
||||
|
||||
with CursorContextManager(filename=output_file) as db:
|
||||
create_tables(db)
|
||||
write_transactions(db, txs)
|
||||
|
||||
|
||||
print("Conversion complete.")
|
Reference in New Issue
Block a user