commit fbd8d11fe50f766a1f91a3c4d645f5f717da7035 Author: Guillem Borrell Date: Wed May 8 21:55:08 2024 +0000 Adding all files diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..b57f1a6 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,62 @@ +name: Build documentation + +on: + push: + branches: + - "main" + +env: + PYTHON_VERSION: 3.9 + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: "pages" + cancel-in-progress: false + +jobs: + build-docs: + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + + steps: + - id: checkout + uses: actions/checkout@v3 + - id: setup-python + uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - id: install-deps-and-build + name: Install dependencies and test + run: | + pip install -e . + pip install -e .[doc] + mkdocs build --site-dir build + + - name: Setup Pages + id: configure-pages + uses: actions/configure-pages@v3 + + - name: Upload artifact + id: upload-artifact + uses: actions/upload-pages-artifact@v1 + with: + # Upload entire repository + path: './build' + + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v2 + + - name: Upload PDF + id: upload-pdf + uses: actions/upload-artifact@v3 + with: + name: document.pdf + path: site/pdf/document.pdf diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e06e50d --- /dev/null +++ b/.gitignore @@ -0,0 +1,163 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + + +*.db \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..8b162db --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,10 @@ +repos: + - repo: https://github.com/psf/black + rev: 22.10.0 + hooks: + - id: black + - repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. + rev: v0.0.278 + hooks: + - id: ruff diff --git a/Caddyfile b/Caddyfile new file mode 100644 index 0000000..51c6e73 --- /dev/null +++ b/Caddyfile @@ -0,0 +1,8 @@ +:80 { + handle_path /api/v1/* { + reverse_proxy localhost:8000 + } + file_server { + root src/retailtwin/api/static + } +} \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..494feff --- /dev/null +++ b/README.md @@ -0,0 +1,43 @@ +# retailtwin + +Implementation of a digital twin of a retail corporation that operates a chain of grocery stores will be described. The implementation is of course limited but comprehensive enough to get some key insights about why corporate IT looks the way it looks. If anyone intends to create antifragile data systems it's important to study how fragile systems come to be on the first place. + +## Bootstrap the database + +First create a postgresql database. + +```bash +createdb -h localhost -U postgresuser retail +``` + +In this case we decided to call the database `retail` and we created it in the same computer we will be running the digital twin. Then sync the data models in the freshly created database with: + +```bash +retailtwin init postgresql://postgresuser:password@localhost/retail +``` + +Then we can populate the database with dummy data with the `bootstrap` subcommand: + +```bash +retailtwin bootstrap postgresql://postgresuser:password@localhost/retail +``` + +Finally we can create all the necessary functions, procedures, and triggers with + +```bash +retailtwin sync postgresql://postgresuser:password@localhost/retail +``` + +## Terminals + +There are currently three available terminals to operate with the digital twin: + +### Stocking terminal + +```bash +stock [DB_URI] [Store location] +``` + +```bash +stock postgresql://postgresuser:password@localhost/retail 1 +``` diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..0514c26 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,3 @@ +# Transactional databases and core applications + +This document is an essay that supports a course \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..adfa77d --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,57 @@ +site_name: Transactional databases and core applications +site_author: Guillem Borrell PhD +site_url: https://git.guillemborrell.es/guillem/PyConES24 +copyright: © Guillem Borrell Nogueras +repo_url: https://git.guillemborrell.es/guillem/PyConES24 +edit_uri: edit/main/docs/ + +nav: + - "Introduction": "index.md" +theme: + name: "material" + palette: + # Palette toggle for light mode + - media: "(prefers-color-scheme)" + primary: teal + toggle: + icon: material/brightness-auto + name: Switch to light mode + + - media: "(prefers-color-scheme: light)" + scheme: default + primary: teal + toggle: + icon: material/brightness-7 + name: Switch to dark mode + + # Palette toggle for dark mode + - media: "(prefers-color-scheme: dark)" + scheme: slate + primary: teal + toggle: + icon: material/brightness-4 + name: Switch to light mode + + features: + - content.action.edit + - content.code.copy + icon: + edit: material/pencil + view: material/eye + +plugins: + - search + +markdown_extensions: + - md_in_html + - admonition + - tables + - pymdownx.highlight: + anchor_linenums: true + line_spans: __span + pygments_lang_class: true + - pymdownx.inlinehilite + - pymdownx.snippets + - pymdownx.superfences + - markdown_include.include: + base_path: docs diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..d5ba508 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,55 @@ +[build-system] +requires = ["flit_core >=3.2,<4"] +build-backend = "flit_core.buildapi" + +[project] +name = "retailtwin" +authors = [{name = "Guillem Borrell", email = "borrell.guillem@bcg.com"}] +readme = "README.md" +dynamic = ["version", "description"] +dependencies = [ + "duckdb", + "pydantic", + "typer", + "rich", + "pyyaml", + "pydantic-settings", + "polars", + "pandas", + "pyarrow", + "sqlalchemy[asyncio] > 2.0.13", + "adbc-driver-postgresql", + "adbc-driver-sqlite", + "prompt_toolkit", + "asyncpg", + "psycopg2-binary", + "pydantic-settings", + "fastapi", + "uvicorn" +] + + +[project.urls] +Home = "https://github.com/Borrell-Guillem_bcgprod/retailtwin" + +[project.optional-dependencies] +doc = [ + "mkdocs-material", + "pymdown-extensions", + "mkdocstrings[python-legacy]>=0.18", + "mkdocs-gen-files", + "markdown-include", + "mkdocs-with-pdf", + "mkdocs-literate-nav"] + +dev = [ + "black", + "ruff", + "pre-commit" +] + +[project.scripts] +retailtwin = "retailtwin.cli.data:app" +stock = "retailtwin.cli.stock:app" +pos = "retailtwin.cli.pos.main:app" +tasks = "retailtwin.cli.tasks.main:app" \ No newline at end of file diff --git a/src/retailtwin/__init__.py b/src/retailtwin/__init__.py new file mode 100644 index 0000000..1185ddb --- /dev/null +++ b/src/retailtwin/__init__.py @@ -0,0 +1,8 @@ +""" +`retailtwin` is a simple digital twin of a grocery store chain. + +Its goal is to showcase the key aspects in automation of enterprise +transactional systems. +""" + +__version__ = '0.0.1' \ No newline at end of file diff --git a/src/retailtwin/api/README.md b/src/retailtwin/api/README.md new file mode 100644 index 0000000..9506d05 --- /dev/null +++ b/src/retailtwin/api/README.md @@ -0,0 +1,30 @@ +# Super simple API-based stock terminal + +Run the backend with + +```bash +DB_URI=postgresql+asyncpg://user:password@server/dbname uvicorn retailtwin.api.main:app +``` + +Use Caddy to reverse proxy and the following Caddyfile. Paths are static so you have to run caddy from the root of the package + +``` +:80 { + handle_path /api/v1/* { + reverse_proxy localhost:8000 + } + file_server { + root src/retailtwin/api/static + } +} +``` + +If you execute + +```bash +caddy run -c Caddyfile +``` + +You should be able to browse the application at http://127.0.0.1, and reach the api docs at http://127.0.0.1/api/v1/docs + +Caddy can be installed in with Chocolatey on Windows, and with Homebrew on mac. \ No newline at end of file diff --git a/src/retailtwin/api/__init__.py b/src/retailtwin/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/retailtwin/api/composition.py b/src/retailtwin/api/composition.py new file mode 100644 index 0000000..e69de29 diff --git a/src/retailtwin/api/db.py b/src/retailtwin/api/db.py new file mode 100644 index 0000000..8c6f34b --- /dev/null +++ b/src/retailtwin/api/db.py @@ -0,0 +1,138 @@ +import os +from operator import itemgetter +from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker, AsyncSession +from sqlalchemy import select, desc +from retailtwin.views import get_inventory_view, get_stores_view +from retailtwin.api.settings import Settings + + +async def get_session(): + try: + engine = create_async_engine(Settings().db_uri.unicode_string()) + yield async_sessionmaker(engine, expire_on_commit=False) + except: # noqa + await engine.dispose() + + +def detail_button(batch: int): + return "".join( + [ + f'' + f'' + f"" + ] + ) + + +async def get_inventory( + store: int, + page: int, + pagesize: int, + sortby: str, + root_path: str, + async_session: async_sessionmaker[AsyncSession] = None, +) -> str: + """Get inventory for a given location + + Args: + store (int): Location id + page (int): Page number for pagination + pagesize (int): Page size for pagination + sortby (str): Column name used for sorting + root_path (str): Root path of the API + async_session (async_sessionmaker[AsyncSession], optional): + SQLAlchemy async session. Defaults to None. + + Returns: + str: HTML table rows with the content + """ + output = list() + inventory = get_inventory_view() + + # Selects the column from which to sort by, and if asc or desc + col_reversed = sortby.startswith("-") + col_selector = itemgetter(sortby.strip("-")) + + async with async_session() as session: + stmt = ( + select( + inventory.c[ + "upc", + "batch", + "name", + "package", + "received", + "best_until", + "quantity", + ] + ) + .where(inventory.c.location == store) + .limit(pagesize) + .offset(pagesize * page) + .order_by( + desc(col_selector(inventory.c)) + if col_reversed + else col_selector(inventory.c) + ) + ) + result = await session.execute(stmt) + + query_params = " ".join( + [ + f'js:{{page: "{page + 1}",' + f'pagesize: "{pagesize}",' + f'sortby: "{sortby}",' + 'store: document.getElementById("store").value}' + ] + ) + + # Turn the results in a sequence of HTML table rows + for i, record in enumerate(result.fetchall()): + line = "" + cells = ( + " ".join(f"{it}" for it in record) + + f" {detail_button(record[1])}" + ) + if i == (pagesize - 1): # Handle last row for infinite scrolling + line = " ".join( + [ + f'{cells}', + ] + ) + else: + line = f"{cells}" + + output.append(line) + + return os.linesep.join(output) + + +async def get_stores( + root_path: str, + async_session: async_sessionmaker[AsyncSession] = None, +): + # Build store options + store_rows = [''] + stores = get_stores_view() + async with async_session() as session: + stmt = select(stores) + result = await session.execute(stmt) + for record in result.fetchall(): + store_rows.append(f'') + + # Wrap with the select tag + query_params = ( + 'js:{page: "0", ' + 'pagesize: "20", ' + 'sortby: "upc", ' + 'store: document.getElementById("store").value}' + ) + return ( + f'