diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 290a717..3c0564a 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,4 +1,4 @@ -# Please see the documentation for all configuration options: +# See the documentation for all configuration options: # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates version: 2 @@ -6,6 +6,8 @@ version: 2 updates: - package-ecosystem: "github-actions" directory: "/" + cooldown: + default-days: 7 schedule: interval: "weekly" day: "sunday" @@ -16,6 +18,8 @@ updates: - package-ecosystem: "pip" directory: "/pkg" + cooldown: + default-days: 7 schedule: interval: "weekly" day: "sunday" @@ -26,6 +30,8 @@ updates: - package-ecosystem: "pip" directory: "/src" + cooldown: + default-days: 7 schedule: interval: "weekly" day: "sunday" diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 120000 index 47dc3e3..0000000 --- a/CLAUDE.md +++ /dev/null @@ -1 +0,0 @@ -AGENTS.md \ No newline at end of file diff --git a/README.md b/README.md index a10ef54..be4a644 100644 --- a/README.md +++ b/README.md @@ -1,95 +1,116 @@ -# Aloha +# Aloha: Python Utilities and Microservice Boilerplate [![License](https://img.shields.io/github/license/LabNow-ai/aloha-python)](https://github.com/LabNow-ai/aloha-python/blob/main/LICENSE) -[![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/LabNow-ai/aloha-python/build.yml?branch=main)](https://github.com/LabNow-ai/aloha-python/actions) -[![PyPI version](https://img.shields.io/pypi/v/aloha)](https://pypi.python.org/pypi/aloha/) -[![PyPI Downloads](https://img.shields.io/pypi/dm/aloha)](https://pepy.tech/badge/aloha/) -[![Code Activity](https://img.shields.io/github/commit-activity/m/LabNow-ai/aloha-python)](https://github.com/LabNow-ai/aloha-python/pulse) -[![Recent Code Update](https://img.shields.io/github/last-commit/LabNow-ai/aloha-python.svg)](https://github.com/LabNow-ai/aloha-python/stargazers) +[![Build](https://img.shields.io/github/actions/workflow/status/LabNow-ai/aloha-python/build.yml?branch=main)](https://github.com/LabNow-ai/aloha-python/actions) +[![PyPI version](https://img.shields.io/pypi/v/aloha)](https://pypi.org/project/aloha/) +[![PyPI downloads](https://img.shields.io/pypi/dm/aloha)](https://pypi.org/project/aloha/) +[![GitHub stars](https://img.shields.io/github/stars/LabNow-ai/aloha-python.svg?label=Stars&style=social)](https://github.com/LabNow-ai/aloha-python/stargazers) -`aloha-python` is a modern project template (boilerplate) and a versatile utility library (`aloha` package) designed to build robust, containerized microservices in Python. +**Aloha is an open-source Python utility library and microservice project template.** It helps Python teams build containerized services with HOCON configuration, structured and multi-process-safe logging, database integrations, encryption utilities, and pytest testing helpers. ---- +Use the installable **`aloha` package** to add common service-building utilities to an existing Python project, or use this repository as a **FastAPI/Uvicorn-oriented microservice boilerplate** with Docker-based development and deployment examples. -Please generously STAR★ our project or donate to us! -[![GitHub Starts](https://img.shields.io/github/stars/LabNow-ai/aloha-python.svg?label=Stars&style=social)](https://github.com/LabNow-ai/aloha-python/stargazers) +- **Documentation:** [English](https://aloha-python.readthedocs.io/en/latest/) · [中文](https://aloha-python.readthedocs.io/zh/latest/) +- **PyPI:** [aloha](https://pypi.org/project/aloha/) +- **Issues and feature requests:** [GitHub Issues](https://github.com/LabNow-ai/aloha-python/issues) -- To understand the package, read the [📚docs](https://aloha-python.readthedocs.io/) or [![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/LabNow-ai/aloha-python) +## Why Aloha? -- To contribute or talk to a human: [![Open an Issue on GitHub](https://img.shields.io/github/issues/LabNow-ai/aloha-python)](https://github.com/LabNow-ai/aloha-python/issues) [![Join the Discord Chat](https://img.shields.io/badge/Discuss_on-Discord-green)](https://discord.gg/kHUzgQxgbJ) [![Join the Gitter Chat](https://img.shields.io/gitter/room/nwjs/nw.js.svg)](https://gitter.im/LabNow-ai/) +- **One library for common service foundations:** configuration, logging, database access, encryption, HTTP service helpers, and tests live in the `aloha` package. +- **Configuration that fits deployments:** load modular HOCON files and apply environment-specific settings through `PROFILE_ENV`, `FILES_CONFIG`, and environment substitutions. +- **Logs built for services:** write plain or JSON logs to console and daily rotating files, with a multi-process-safe file handler. +- **Database integrations:** use SQLAlchemy-backed operators and helpers for PostgreSQL, MySQL, SQLite, DuckDB, MongoDB, Redis, Elasticsearch, Oracle, and Kafka. +- **A practical microservice starting point:** the repository includes FastAPI/Uvicorn service examples, Dockerfiles, Docker Compose development setup, and pytest examples. +- **Optional binary builds:** compile Python modules to native extensions with the `aloha compile` command and Cython. ---- +## Install -## 🚀 Key Features +The base package supports Python 3.10 and later: -- **Configuration Management (`aloha.config`)**: Lazy-loaded settings (`SETTINGS`) using HOCON (Human-Optimized Config Object Notation), supporting environment profile overrides (`PROFILE_ENV` / `FILES_CONFIG`, with legacy `ENV_PROFILE` deprecated) and environment variable injection. -- **Concurrent-Safe Logging (`aloha.logger`)**: Multi-process safe daily rotating log file handler, console output, and automatic log paths configuration. -- **Database Operators (`aloha.db`)**: Pre-built SQLAlchemy-backed connections for PostgreSQL, MySQL, SQLite, DuckDB, MongoDB, Redis, Elasticsearch, and Kafka, with password resolution via a secure `PasswordVault` wrapper. -- **Encryption & Utilities (`aloha.encrypt`)**: Fast helpers for AES (ECB/CBC) encryption, RSA asymmetric key-pair generation/signatures, JWT encoding/decoding, and Base62 hashing. -- **Testing Harness (`aloha.testing`)**: Extended `UnitTestCase` and integration `ServiceTestCase` structures for testing HTTP endpoints. -- **Binary Code Protection (`aloha compile`)**: Utility compiler using Cython to package python source files (`.py`) into platform-native compiled dynamic libraries (`.so`/`.pyd`), protecting intellectual property. - ---- +```bash +python -m pip install aloha +``` -## 📁 Repository Layout +Install optional dependency groups for the capabilities you need: -- **[`src/`](src)**: Application-specific codebase, configuration files (`src/resource/config/`), and unit/integration tests (`src/tests/`). -- **[`pkg/`](pkg)**: The core `aloha` utility library source code. -- **[`tool/`](tool)**: Local CI/CD files and setup helpers (e.g. docker-compose configurations, lifecycle scripts). -- **[`doc/`](doc)**: Documentation files and AI Agent Skills. -- **[`notebook/`](notebook)**: Jupyter notebooks for interactive analysis. +```bash +python -m pip install "aloha[service]" # FastAPI and Uvicorn helpers +python -m pip install "aloha[db]" # Database integrations +python -m pip install "aloha[all]" # All optional integrations and tools +``` ---- +## Quick Start -## 🛠️ Local Development Quick Start +Check the installed library version: -Local development is fully containerized using Docker & Docker Compose to maintain environment consistency. +```python +from aloha import __version__ -### 1. Launch Dev Container +print(__version__) +``` -Run the lifecycle helper script: +Use the shared application logger after configuring the service's HOCON settings: -```bash -# Check port availability and spin up the development container -./tool/cicd/run-dev.sh up +```python +from aloha.logger import LOG -# Open an interactive terminal inside the container -./tool/cicd/run-dev.sh enter +LOG.info("Service is ready") ``` -_Note: Development ports (`PORT_APP` and `PORT_WEB`) are dynamically computed based on your User ID (`UID`) to avoid conflicts on shared servers._ +The global logger reads `deploy.log_level`, `deploy.log_format_file`, and `deploy.log_format_stream` from the loaded configuration. File logs default to JSON; console logs default to plain text. See the [logging guide](doc/skills/aloha_python/references/logger.md) for formats, fields, timestamps, and configuration details. -### 2. Run Tests +## What Is Included? -Inside the running container terminal: +| Module | What it helps with | +| --------------- | ------------------------------------------------------------------------------------------------------ | +| `aloha.config` | Load HOCON settings, profiles, and environment-based configuration. | +| `aloha.logger` | Configure global and named loggers, JSON/plain output, and daily rotating logs. | +| `aloha.db` | Connect to supported SQL and service databases with reusable operators and password-vault integration. | +| `aloha.encrypt` | Use AES, RSA, JWT, hashing, and password-vault helpers. | +| `aloha.service` | Build FastAPI/Uvicorn services and reusable HTTP API handlers. | +| `aloha.testing` | Test utilities for unit tests and service API tests. | +| `aloha compile` | Build selected Python modules as native extensions using Cython. | -```bash -# Execute pytest suite -pytest src/ +Explore the [API documentation](https://aloha-python.readthedocs.io/en/main/api/) or the detailed [Aloha Python Skill guide](doc/skills/aloha_python/SKILL.md). -# Run tests with code coverage report -pytest --cov=src src/ +## Use This Repository as a Service Template + +The repository includes a runnable application example under `src/`, modular configuration under `src/resource/config/`, and Docker-based development helpers. + +Start the development container from the repository root: + +```bash +./tool/cicd/run-dev.sh up +./tool/cicd/run-dev.sh enter ``` -### 3. Production Packaging +Inside the container, run the example service and tests: + +```bash +cd /workspace/src +python3 main.py app_common.main +pytest ./ +``` -To build a production-ready Docker image with optional binary compilation enabled: +To build a production Docker image from the repository root: ```bash source tool/tool.sh build_image app_common latest src/app-demo.Dockerfile ``` ---- +See the [development and scaffolding guide](doc/skills/aloha_cicd/SKILL.md) for container lifecycle, configuration, ports, and production builds. -## ✍️ Coding Guidelines +## Repository Layout -When developing in this project, variables should place their **type or primary characteristics/role prefix first**: +- `pkg/` — source code and packaging metadata for the installable `aloha` library. +- `src/` — example application, HOCON configuration, and tests. +- `tool/` — Docker, Compose, and development lifecycle scripts. +- `doc/` — user documentation, API references, and contributor skills. +- `notebook/` — notebooks for interactive examples and experiments. -- _Correct_: `name_service`, `port_service`, `svc_ingress`, `cfg_postgres`. -- _Incorrect_: `service_name`, `service_port`, `ingress_service`, `postgres_config`. +## Contributing -For details on local setups, CI/CD specifications, and sub-module APIs, inspect our agent instruction files: +Before changing code, review the repository [contributor guidelines](AGENTS.md) for coding, configuration, logging, and testing standards. See the [Aloha Python Skill](doc/skills/aloha_python/SKILL.md) for package development and the [Aloha CI/CD & Scaffolding Skill](doc/skills/aloha_cicd/SKILL.md) for local development and builds. -- **[Aloha Python Skills Guide](doc/skills/aloha_python/SKILL.md)** -- **[Aloha CI/CD & Scaffolding Guide](doc/skills/aloha_cicd/SKILL.md)** +Contributions, bug reports, and feature requests are welcome via [GitHub Issues](https://github.com/LabNow-ai/aloha-python/issues). diff --git a/doc/.readthedocs.yaml b/doc/.readthedocs.yaml index ecc4d79..fb89b61 100644 --- a/doc/.readthedocs.yaml +++ b/doc/.readthedocs.yaml @@ -1,6 +1,6 @@ # .readthedocs.yaml -# Read the Docs configuration file -# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details +# Read the Docs configuration for the bilingual MkDocs site. +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details. version: 2 @@ -10,12 +10,12 @@ build: python: "3" jobs: pre_create_environment: - - echo "Command run at 'pre_create_environment' step" - - 'cp ./README.md ./pkg/' + - cp ./README.md ./pkg/ pre_build: - - 'case "${READTHEDOCS_LANGUAGE}" in zh*) cp ./doc/mkdocs.zh.yml ./doc/mkdocs.yml ;; esac' - post_build: - - echo "Command run at 'post_build' step $(date)" + - | + case "${READTHEDOCS_LANGUAGE:-en}" in + zh*) cp ./doc/mkdocs.zh.yml ./doc/mkdocs.yml ;; + esac mkdocs: configuration: ./doc/mkdocs.yml diff --git a/doc/en/README-config.md b/doc/en/README-config.md index 94e776f..5907023 100644 --- a/doc/en/README-config.md +++ b/doc/en/README-config.md @@ -36,6 +36,18 @@ Define the application module name. It is mapped to config variable `APP_MODULE` Define where log files are stored. +### `deploy.log_format_file` + +*Default value*: `json`. + +Select the predefined format for log files. Supported values are `plain` and `json`. + +### `deploy.log_format_stream` + +*Default value*: `plain`. + +Select the predefined format for terminal streams (stderr/stdout). Supported values are `plain` and `json`; the plain format colors the level prefix when writing to a TTY. + ### `DIR_RESOURCE` *Default value*: `resource` under the current working directory. diff --git a/doc/en/README-get-start.md b/doc/en/README-get-start.md index 168968d..2fa4212 100644 --- a/doc/en/README-get-start.md +++ b/doc/en/README-get-start.md @@ -1,116 +1,99 @@ -# Getting Started +# Getting Started with Aloha -## Step 1. Install the Python package +This guide covers two ways to use Aloha: -```title="Install aloha with all extra requirements" -pip install aloha[all] -``` - -## Step 2. Use this repository as a boilerplate +1. Install the `aloha` Python library in an existing application. +2. Use the `aloha-python` repository as a Docker-based microservice template. -This repository serves as a boilerplate/template project built on top of `aloha`. -It gives you a ready-to-use application layout, development scripts, and containerized tooling so you can start building instead of assembling the project skeleton yourself. +## 1. Install the Python Library -### What this template gives you +Aloha supports Python 3.10 and later. Install the base package: -- A containerized development environment based on Docker and Docker Compose -- Pre-installed Python and project dependencies -- An application entry point you can extend directly -- A conventional layout for source code, documentation, notebooks, and tooling +```bash +python -m pip install aloha +``` -### Recommended workflow +Install extras for optional capabilities: -1. Clone this repository. -2. Inspect the starter application structure. -3. Use the scripts under `tool/cicd/` to start the development container when you want a reproducible environment. -4. Put your own application code in the template structure and grow from there. +```bash +python -m pip install "aloha[service]" # FastAPI and Uvicorn support +python -m pip install "aloha[db]" # Database integrations +python -m pip install "aloha[all]" # All optional dependencies +``` -### Launch the development environment +Other extras include `aio` for asynchronous database drivers, `build` for Cython compilation, `stream` for Kafka clients, `data` and `report` for data/reporting tools, and `test` and `docs` for development. -If you want the full boilerplate experience, start the containerized DEV environment: +Verify the installation: ```bash -./tool/cicd/run-dev.sh up -./tool/cicd/run-dev.sh enter +aloha info ``` -The `up` command creates or starts the development container. The `enter` command opens an interactive shell inside that container. +## 2. Use Aloha in an Application -### Project structure +Import only the modules your application needs. For example, the shared logger is available from `aloha.logger`: -The template is organized around a few common folders: - -- `doc/`: Documentation source files. You can place your project's documentation here. -- `src/`: Application code and entry points. This is where your business logic goes. It contains a demo application (`app_common`) showing how to use `aloha`. -- `tool/`: Scripts and Docker assets for development and CI/CD. +```python +from aloha.logger import LOG -### How to use `aloha` in your project +LOG.info("Application started") +``` -The `src/` directory contains a demo application that demonstrates how to use the `aloha` package. Here is a brief overview of how to import and use `aloha` for regular Python project development: +To use HOCON settings, provide a resource configuration directory and load values from `SETTINGS`: -1. **Import `aloha` modules**: Import the necessary `aloha` modules for your application logic. For example, `aloha.logger` for logging, `aloha.config` for configuration, or `aloha.db` for database interactions. +```python +from aloha.settings import SETTINGS -2. **Configure your application**: Define your application configuration in `src/resource/config/main.conf`. +config_settings = SETTINGS.config +``` -3. **Implement your application logic**: Write your Python code using the imported `aloha` modules. +By default, Aloha looks for configuration under `resource/config/` relative to the current working directory. `PROFILE_ENV`, `FILES_CONFIG`, `DIR_RESOURCE`, and `DIR_CONFIG` can be used to select profiles and locations. See the [configuration guide](README-config.md) for details. -4. **Run your application**: You can run your application using the provided `main.py` script: +The `aloha start` command can run a module whose `main()` function starts your application: ```bash -python3 src/main.py your_module.main +aloha start your_package.main ``` -This command tells `src/main.py` to find and execute the `main()` function within your specified module (e.g., `your_module.main`). - -## Configuration with HOCON - -`aloha` uses the HOCON (Human-Optimized Config Object Notation) format for its configuration files. This allows for hierarchical, modular, and human-readable configurations. +## 3. Start from the Microservice Template -### Configuration File Location +Clone the repository and enter the project: -By default, `aloha` looks for configuration files in the `src/resource/config/` directory. The primary configuration file is `main.conf`. - -### Modular Configuration - -HOCON supports including other configuration files, which is useful for modularizing your settings (e.g., separating development, staging, and production configurations). - -**Example `main.conf`:** - -```hocon -include "deploy-DEV.conf" - -app_name = "MyAlohaApp" +```bash +git clone https://github.com/LabNow-ai/aloha-python.git +cd aloha-python +``` -server { - host = "0.0.0.0" - port = 8080 -} +Start and enter the containerized development environment: -database { - type = "postgresql" - connection_string = "${?DB_CONNECTION_STRING}" # Environment variable override -} +```bash +./tool/cicd/run-dev.sh up +./tool/cicd/run-dev.sh enter ``` -In this example: -- `include "deploy-DEV.conf"` brings in settings from another file, allowing for environment-specific overrides. -- `app_name` and `server` define application-specific settings. -- `database.connection_string = "${?DB_CONNECTION_STRING}"` demonstrates how to use environment variables to override configuration values, making it flexible for different deployment environments. +From the container shell, run the sample FastAPI/Uvicorn service and tests: -### Accessing Configuration in Code +```bash +cd /workspace/src +python3 main.py app_common.main +pytest ./ +``` -You can access configuration values in your Python code via `aloha.settings.SETTINGS.config`: +The sample application lives in `src/app_common/`; service configuration and profiles live in `src/resource/config/`. Adapt these to your application. See the [development guide](README-develop.md) for container commands and [configuration guide](README-config.md) for HOCON setup. -```python -from aloha.settings import SETTINGS +## 4. Build a Production Image -app_name = SETTINGS.config.get("app_name") -server_port = SETTINGS.config.get("server.port") +From the repository root, build the example Docker image: -print(f"Application Name: {app_name}") -print(f"Server Port: {server_port}") +```bash +source tool/tool.sh +build_image app_common latest src/app-demo.Dockerfile ``` -This approach ensures that your application remains configurable and adaptable to various deployment scenarios. +The Docker build can compile selected application modules with Cython. See the [CLI guide](README-cli.md) and [compilation reference](../skills/aloha_python/references/compile.md). + +## Next Steps -[:octicons-mark-github-16: Go to Template Project](https://github.com/LabNow-ai/aloha-python/tree/main/src){ .md-button } +- Browse the [API reference](api/index.md). +- Learn about [structured logging](api/logging.md), [database integrations](api/db.md), and [HOCON configuration](README-config.md). +- Review the [12-Factor application guide](README-12factor.md) for deployment practices. diff --git a/doc/en/index.md b/doc/en/index.md index b23a294..666ddc1 100644 --- a/doc/en/index.md +++ b/doc/en/index.md @@ -1,42 +1,87 @@ -# Introduction +# Aloha: Python Microservice Library and Project Template -Aloha! Thanks for your interest in this Python package. +**Aloha is an open-source Python utility library and microservice boilerplate for building containerized services.** The `aloha` package brings together HOCON configuration, structured logging, database operators, encryption utilities, FastAPI/Uvicorn service helpers, and testing support. +Use Aloha as a dependency in an existing Python application, or use the `aloha-python` repository as a Docker-ready starting point for a new microservice. + +[![PyPI version](https://img.shields.io/pypi/v/aloha)](https://pypi.org/project/aloha/) +[![Build](https://img.shields.io/github/actions/workflow/status/LabNow-ai/aloha-python/build.yml?branch=main)](https://github.com/LabNow-ai/aloha-python/actions) [![License](https://img.shields.io/github/license/LabNow-ai/aloha-python)](https://github.com/LabNow-ai/aloha-python/blob/main/LICENSE) -[![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/LabNow-ai/aloha-python/build.yml?branch=main)](https://github.com/LabNow-ai/aloha-python/actions) -[![Join the Gitter Chat](https://img.shields.io/gitter/room/nwjs/nw.js.svg)](https://gitter.im/LabNow-ai/) -[![PyPI version](https://img.shields.io/pypi/v/aloha)](https://pypi.python.org/pypi/aloha/) -[![PyPI Downloads](https://img.shields.io/pypi/dm/aloha)](https://pepy.tech/badge/aloha/) -[![Code Activity](https://img.shields.io/github/commit-activity/m/LabNow-ai/aloha-python)](https://github.com/LabNow-ai/aloha-python/pulse) -[![Recent Code Update](https://img.shields.io/github/last-commit/LabNow-ai/aloha-python.svg)](https://github.com/LabNow-ai/aloha-python/stargazers) -Please generously STAR our project or donate to us! [![GitHub Stars](https://img.shields.io/github/stars/LabNow-ai/aloha-python.svg?label=Stars&style=social)](https://github.com/LabNow-ai/aloha-python/stargazers) +## What Aloha Provides + +- **HOCON configuration** — Load modular configuration files and select profiles with environment variables. +- **Service-ready logging** — Emit plain or JSON logs to the console and daily rotating files, including process and host metadata. +- **Database connectivity** — Reusable operators and helpers for PostgreSQL, MySQL, SQLite, DuckDB, MongoDB, Redis, Elasticsearch, Oracle, and Kafka. +- **Security utilities** — AES and RSA operations, JWT helpers, hashing, and password-vault integrations. +- **Python web services** — FastAPI/Uvicorn application helpers and reusable HTTP service components. +- **Testing and packaging tools** — Test helpers and an optional Cython-based command for building selected modules as native extensions. +- **Containerized template** — A sample application, Dockerfiles, Docker Compose development environment, and CI/CD scripts. + +## Install the Python Package + +Aloha requires Python 3.10 or later. Install the base package with pip: + +```bash +python -m pip install aloha +``` + +Optional integrations are grouped as extras. Install only what your application needs: + +```bash +python -m pip install "aloha[service]" # FastAPI and Uvicorn +python -m pip install "aloha[db]" # Database integrations +python -m pip install "aloha[all]" # All optional integrations and tools +``` + +See [Installation and first steps](README-get-start.md) for more extras and a minimal usage example. + +## Choose Your Path -The Python package `aloha` is a versatile toolkit for building Python microservices. -It encapsulates commonly used components and features, such as: +### Add Aloha to an Existing Python Project -- Rapidly creating RESTful APIs and starting services -- Logging utilities -- Managing environments, configuration files, and resource files -- Connecting to popular databases -- Detecting and monitoring runtime environments +Install the package, configure settings with HOCON, and import the modules you need: -## Installation +```python +from aloha.logger import LOG -```title="Install aloha with extra requirements" -pip install aloha[all] +LOG.info("Service is ready") ``` -Notice that `[all]` after the package name is a set of extra requirements that enable additional features. +The global logger reads its level and file/console formats from the `deploy` HOCON settings. See the [configuration guide](README-config.md) and [logging reference](api/logging.md). + +### Start from the Microservice Template + +Clone the repository and start its development container: + +```bash +git clone https://github.com/LabNow-ai/aloha-python.git +cd aloha-python +./tool/cicd/run-dev.sh up +./tool/cicd/run-dev.sh enter +``` + +Inside the container: + +```bash +cd /workspace/src +python3 main.py app_common.main +pytest ./ +``` + +The `src/app_common/` application is an example to adapt for your service. Follow the [Getting Started guide](README-get-start.md) for setup and project structure. + +## Documentation + +- [Getting Started](README-get-start.md) +- [Configuration and HOCON profiles](README-config.md) +- [CLI commands](README-cli.md) +- [12-Factor application guide](README-12factor.md) +- [Development and Docker guide](README-develop.md) +- [API reference](api/index.md) -These extras include: +## Project and Community -- `all`: includes everything listed below -- `service`: packages used to build RESTful APIs (`aloha` uses Tornado for services) -- `build`: compile Python code into binary files, useful for source code protection -- `db`: connect to popular databases, such as MySQL / PostgreSQL / Redis -- `stream`: process stream data using `confluent_kafka` -- `data`: process data or do data science tasks using packages like `pandas` -- `report`: export data and reports to Excel files -- `test`: unit test utilities -- `docs`: documentation build utilities +- [Source code on GitHub](https://github.com/LabNow-ai/aloha-python) +- [PyPI package](https://pypi.org/project/aloha/) +- [Report a bug or request a feature](https://github.com/LabNow-ai/aloha-python/issues) diff --git a/doc/mkdocs.yml b/doc/mkdocs.yml index be56504..a0645e9 100644 --- a/doc/mkdocs.yml +++ b/doc/mkdocs.yml @@ -1,10 +1,12 @@ -site_name: Aloha - a versatile Python library to build microservice. +site_name: Aloha Python Microservice Library +site_description: Open-source Python utilities and microservice boilerplate for HOCON configuration, structured logging, database integrations, encryption, FastAPI, and Docker-based development. repo_url: https://github.com/LabNow-ai/aloha-python edit_uri: blob/main/doc/en/ site_url: !ENV READTHEDOCS_CANONICAL_URL docs_dir: ./en theme: name: material + language: en highlightjs: true features: - navigation.tabs diff --git a/doc/mkdocs.zh.yml b/doc/mkdocs.zh.yml index 11d8c0d..3f0aa15 100644 --- a/doc/mkdocs.zh.yml +++ b/doc/mkdocs.zh.yml @@ -1,10 +1,12 @@ -site_name: Aloha - a versatile Python library to build microservice. +site_name: Aloha Python 微服务工具库 +site_description: 开源 Python 工具库与微服务模板,提供 HOCON 配置、结构化日志、数据库集成、加密工具、FastAPI 和 Docker 开发环境。 repo_url: https://github.com/LabNow-ai/aloha-python edit_uri: blob/main/doc/zh/ site_url: !ENV READTHEDOCS_CANONICAL_URL docs_dir: ./zh theme: name: material + language: zh highlightjs: true features: - navigation.tabs diff --git a/doc/skills/aloha_cicd/references/dev_env.md b/doc/skills/aloha_cicd/references/dev_env.md index 0d0f089..8cc5e11 100644 --- a/doc/skills/aloha_cicd/references/dev_env.md +++ b/doc/skills/aloha_cicd/references/dev_env.md @@ -23,8 +23,9 @@ To guarantee environment consistency, local development is fully containerized. - **User-Specific Port Mapping**: To prevent port clashes on shared, multi-user systems, host ports are computed dynamically using the user's numeric ID (`UID`): - **Application Port (`PORT_APP`)**: `30000 + UID` (exposes port 9000). - **Web Port (`PORT_WEB`)**: `33000 + UID` (exposes port 3000). -- **Import Environments**: The container mounts the project root to `/root/app:rw` and automatically sets `PYTHONPATH=/root/app/pkg:/root/app/src:/root/app/notebook` for clean imports. -- **Agent Credentials Mounts**: Volume-mounts local config directories (`~/.gemini`, `~/.claude`, `~/.copilot`) to keep credential cache and auth profiles active. +- **Container User**: `run-dev.sh` reads the host UID and GID with `id -u` and `id -g`, then passes them to Compose. The container runs with that numeric UID/GID so files created in the workspace are owned by the host user. No `.env` file is needed; run the environment through `run-dev.sh`. +- **Workspace and Imports**: The container mounts the project root to `/workspace:rw` and sets `PYTHONPATH=/workspace/pkg:/workspace/src:/workspace/notebook` for clean imports. +- **Agent Credentials Mounts**: Local config directories (`~/.gemini`, `~/.claude`, `~/.copilot`, `~/.codex`) are mounted under `/home/` and `HOME` is set to that path so the non-root container user can access them. --- diff --git a/doc/skills/aloha_python/references/logger.md b/doc/skills/aloha_python/references/logger.md index fff368c..795ca3a 100644 --- a/doc/skills/aloha_python/references/logger.md +++ b/doc/skills/aloha_python/references/logger.md @@ -4,7 +4,23 @@ The `aloha.logger` subpackage provides a pre-configured, multi-process safe logg ## 1. Global Logger (`LOG`) -The module exports a global pre-configured logger named `LOG`. It reads log levels from settings under `deploy.log_level` (falling back to `logging.DEBUG` if unset). +The module exports a global pre-configured logger named `LOG`. When `aloha.logger` is imported, it reads these settings from the `deploy` HOCON object: + +- `deploy.log_level` sets the minimum severity level; it accepts a Python logging level name such as `"INFO"` or a numeric level, and defaults to `DEBUG` when unset. This controls which records are emitted; it does **not** select the log format. +- `deploy.log_format_file` selects the file format and defaults to `"json"`. +- `deploy.log_format_stream` selects the console format and defaults to `"plain"`. + +For example, this configuration logs INFO and higher records, writes JSON lines to the log file, and uses plain text on the console: + +```hocon +deploy = { + log_level = "INFO" + log_format_file = "json" + log_format_stream = "plain" +} +``` + +The available formats and their output fields are described in [Predefined formats](#predefined-formats). The global `LOG` is configured at import time; set these values in the loaded HOCON configuration before importing `aloha.logger`. ### Usage Example @@ -44,12 +60,36 @@ logger_custom.info("Custom sync logger initialized.") ## 3. Implementation Details +### Predefined formats + +The global logger's HOCON format settings are shown above. `deploy.log_format_file` and `deploy.log_format_stream` each accept `"plain"` or `"json"`: + +- `plain`: the default format, with an aligned level prefix, ISO 8601 timestamp with millisecond precision and the host's configured local UTC offset (the same representation as the JSON `timestamp` field), `filename:lineno` source location, and message. The console handler colors the level prefix by severity when its stream is a terminal; log files remain uncolored. +- `json`: one JSON object per line with `timestamp`, `level`, `logger`, `pid`, `hostname`, `source` (`filename:lineno`), and `message` fields. `pid` and `hostname` identify the current Python process and runtime host. Exception information is included when available. WARNING and higher records also include a captured calling stack in `stack_info`. + +The file format defaults to `json`; the console format defaults to `plain`. The console formatter adds colors when its output stream is a terminal. The deprecated `log_format` argument to `setup_logger` and `get_logger` remains available as an explicit override for both handlers. `formatter_str`, when provided, takes precedence over the selected formats for both handlers. + +- Aloha and FastAPI/Uvicorn ordinary logs share the root logger's stderr handler and one ordinary log file. Ordinary Uvicorn loggers propagate to the root instead of creating duplicate files. +- Uvicorn access logs write to stdout and a separate `_.access.log` file. Their file/console formats use the same independent settings; in plain mode they use Uvicorn's access format without a source location and append `pid`, `hostname`, and additional `LogRecord` fields as a JSON object. In JSON mode the `source` and `logger` fields are omitted, while `pid`, `hostname`, and additional `LogRecord` fields are included. + +### Parsing timestamps + +Log timestamps are ISO 8601 strings with the runtime host's UTC offset, for example `2026-09-27T23:20:41.353+08:00`. Parse them as timezone-aware values to preserve the represented instant. + +In Python, `datetime.fromisoformat()` parses the offset into an aware `datetime`: + +```python +from datetime import datetime + +timestamp_log = datetime.fromisoformat("2026-09-27T23:20:41.353+08:00") +``` + +In DuckDB, cast the string to `TIMESTAMPTZ`: + +```sql +SELECT CAST('2026-09-27T23:20:41.353+08:00' AS TIMESTAMPTZ) AS timestamp_log; +``` + - **Safe Concurrent File Writes**: Utilizes `MultiProcessSafeDailyRotatingFileHandler` to avoid lock conflicts or log corruption when multiple parallel processes write logs concurrently. - **Log Location**: Writes logs to the directory specified by the `DIR_LOG` environment variable (defaults to `logs/`). -- **File Naming Format**: Log file names include: - - Application module (`APP_MODULE`) - - Logger name - - Hostname - - PID (Process ID) - - Example: `app_module_default_hostname_p12345.log` +- **File Naming Format**: Ordinary log files use `_.log`; access log files use `_.access.log`. If `APP_MODULE` is unset or empty, `default` is used. For example: `Aloha_2026-0927.log` and `Aloha_2026-0927.access.log`. diff --git a/doc/zh/README-config.md b/doc/zh/README-config.md index 73a239e..4a11dbd 100644 --- a/doc/zh/README-config.md +++ b/doc/zh/README-config.md @@ -36,6 +36,18 @@ 用于定义日志文件存储目录。 +### `deploy.log_format_file` + +*默认值*:`json`。 + +用于选择日志文件的预定义格式。支持 `plain` 和 `json`。 + +### `deploy.log_format_stream` + +*默认值*:`plain`。 + +用于选择终端(stderr/stdout)日志的预定义格式。支持 `plain` 和 `json`;TTY 终端中的 plain 格式会按日志级别着色。 + ### `DIR_RESOURCE` *默认值*:当前工作目录下的 `resource`。 diff --git a/doc/zh/README-get-start.md b/doc/zh/README-get-start.md index 17db4e9..976fce4 100644 --- a/doc/zh/README-get-start.md +++ b/doc/zh/README-get-start.md @@ -1,116 +1,99 @@ -# 快速开始 +# Aloha 快速开始 -## 第一步:安装 Python 包 +本指南介绍两种使用 Aloha 的方式: -```title="安装 aloha 及全部扩展依赖" -pip install aloha[all] -``` - -## 第二步:把本仓库当作 boilerplate 使用 +1. 在现有应用中安装和使用 `aloha` Python 工具库。 +2. 将 `aloha-python` 仓库作为基于 Docker 的微服务模板。 -本仓库是一个基于 `aloha` 的样板/模板项目。 -它提供了一个即用的应用程序布局、开发脚本和容器化工具,让你无需从头搭建项目骨架,即可开始构建。 +## 1. 安装 Python 工具库 -### 这个模板提供了什么 +Aloha 支持 Python 3.10 及以上版本。安装基础包: -- 基于 Docker 和 Docker Compose 的容器化开发环境 -- 预装好的 Python 运行环境和项目依赖 -- 可直接扩展的应用入口 -- 适合持续开发的常见目录结构:源码、文档、Notebook、工具脚本 +```bash +python -m pip install aloha +``` -### 推荐使用方式 +按需安装可选能力: -1. 克隆本仓库。 -2. 检查启动应用程序的结构。 -3. 需要可复现开发环境时,使用 `tool/cicd/` 里的脚本启动开发容器。 -4. 在模板结构上放入你自己的业务代码,并逐步扩展。 +```bash +python -m pip install "aloha[service]" # FastAPI 和 Uvicorn 支持 +python -m pip install "aloha[db]" # 数据库集成 +python -m pip install "aloha[all]" # 所有可选依赖 +``` -### 启动开发环境 +其他 extras 包括:`aio` 异步数据库驱动、`build` Cython 编译、`stream` Kafka 客户端、`data` 和 `report` 数据处理与报表工具,以及用于开发的 `test` 和 `docs`。 -如果你想直接使用完整的 boilerplate 开发环境,可以启动容器化 DEV 环境: +验证安装: ```bash -./tool/cicd/run-dev.sh up -./tool/cicd/run-dev.sh enter +aloha info ``` -其中 `up` 命令用于创建或启动开发容器,`enter` 命令用于进入容器内部的交互式 Shell。 +## 2. 在应用中使用 Aloha -### 项目结构 +只需导入应用所需模块。例如,从 `aloha.logger` 导入共享 logger: -这个模板围绕几个常见目录组织: - -- `doc/`:文档源文件。你可以在这里放置你的项目文档。 -- `src/`:应用程序代码和入口。这是你的业务逻辑所在。它包含一个演示应用程序 (`app_common`),展示了如何使用 `aloha`。 -- `tool/`:开发和 CI/CD 相关的脚本和 Docker 资源。 +```python +from aloha.logger import LOG -### 如何在你的项目中使用 `aloha` 包 +LOG.info("Application started") +``` -`src/` 目录包含一个演示应用程序,展示了如何使用 `aloha` 包。以下是如何在常规 Python 项目开发中导入和使用 `aloha` 的简要概述: +如需使用 HOCON 配置,可指定资源配置目录并通过 `SETTINGS` 读取: -1. **导入 `aloha` 模块**:为你的应用程序逻辑导入必要的 `aloha` 模块。例如,`aloha.logger` 用于日志记录,`aloha.config` 用于配置,或 `aloha.db` 用于数据库交互。 +```python +from aloha.settings import SETTINGS -2. **配置你的应用程序**:在 `src/resource/config/main.conf` 中定义你的应用程序配置。 +config_settings = SETTINGS.config +``` -3. **实现你的应用程序逻辑**:使用导入的 `aloha` 模块编写你的 Python 代码。 +默认情况下,Aloha 会相对于当前工作目录从 `resource/config/` 查找配置。可以通过 `PROFILE_ENV`、`FILES_CONFIG`、`DIR_RESOURCE` 和 `DIR_CONFIG` 选择配置 profile 与配置文件位置。详情见[配置指南](README-config.md)。 -4. **运行你的应用程序**:你可以使用提供的 `main.py` 脚本运行应用程序: +`aloha start` 可以运行包含 `main()` 函数的应用模块: ```bash -python3 src/main.py your_module.main +aloha start your_package.main ``` -此命令指示 `src/main.py` 查找并执行你指定模块(例如 `your_module.main`)中的 `main()` 函数。 - -## HOCON 配置 - -`aloha` 使用 HOCON (Human-Optimized Config Object Notation) 格式作为其配置文件。这允许分层、模块化和人类可读的配置。 +## 3. 从微服务模板开始 -### 配置文件位置 +克隆仓库并进入项目目录: -默认情况下,`aloha` 会在 `src/resource/config/` 目录中查找配置文件。主配置文件是 `main.conf`。 - -### 模块化配置 - -HOCON 支持包含其他配置文件,这对于模块化你的设置非常有用(例如,分离开发、预发布和生产环境的配置)。 - -**`main.conf` 示例:** - -```hocon -include "deploy-DEV.conf" - -app_name = "MyAlohaApp" +```bash +git clone https://github.com/LabNow-ai/aloha-python.git +cd aloha-python +``` -server { - host = "0.0.0.0" - port = 8080 -} +启动并进入容器化开发环境: -database { - type = "postgresql" - connection_string = "${?DB_CONNECTION_STRING}" # 环境变量覆盖 -} +```bash +./tool/cicd/run-dev.sh up +./tool/cicd/run-dev.sh enter ``` -在此示例中: -- `include "deploy-DEV.conf"` 引入了来自另一个文件的设置,允许进行特定于环境的覆盖。 -- `app_name` 和 `server` 定义了应用程序特定的设置。 -- `database.connection_string = "${?DB_CONNECTION_STRING}"` 演示了如何使用环境变量覆盖配置值,使其适用于不同的部署环境。 +在容器 Shell 中运行 FastAPI/Uvicorn 示例服务和测试: -### 在代码中访问配置 +```bash +cd /workspace/src +python3 main.py app_common.main +pytest ./ +``` -你可以通过 `aloha.settings.SETTINGS.config` 在你的 Python 代码中访问配置值: +示例应用位于 `src/app_common/`;服务配置和 profile 位于 `src/resource/config/`。可以基于这些示例添加自己的业务逻辑。容器命令见[开发指南](README-develop.md),HOCON 配置见[配置指南](README-config.md)。 -```python -from aloha.settings import SETTINGS +## 4. 构建生产镜像 -app_name = SETTINGS.config.get("app_name") -server_port = SETTINGS.config.get("server.port") +在仓库根目录执行以下命令,构建示例 Docker 镜像: -print(f"Application Name: {app_name}") -print(f"Server Port: {server_port}") +```bash +source tool/tool.sh +build_image app_common latest src/app-demo.Dockerfile ``` -这种方法确保了你的应用程序可配置并适应各种部署场景。 +Docker 构建过程可以使用 Cython 编译指定的应用模块。详情见[CLI 指南](README-cli.md)和[编译参考](../skills/aloha_python/references/compile.md)。 + +## 后续阅读 -[:octicons-mark-github-16: 前往模板项目](https://github.com/LabNow-ai/aloha-python/tree/main/src){ .md-button } +- 浏览 [API 参考](api/index.md)。 +- 了解[结构化日志](api/logging.md)、[数据库集成](api/db.md)和 [HOCON 配置](README-config.md)。 +- 查看 [12-Factor 应用指南](README-12factor.md),了解部署实践。 diff --git a/doc/zh/index.md b/doc/zh/index.md index 97019da..a0d8976 100644 --- a/doc/zh/index.md +++ b/doc/zh/index.md @@ -1,41 +1,87 @@ -# 介绍 +# Aloha:Python 微服务工具库与项目模板 -Aloha!感谢关注这个 Python 包。 +**Aloha 是一个开源 Python 工具库和微服务项目模板,帮助开发者构建可容器化部署的服务。** `aloha` 包集成了 HOCON 配置、结构化日志、数据库操作、加密工具、FastAPI/Uvicorn 服务辅助组件和测试支持。 -[![License](https://img.shields.io/github/license/LabNow-ai/aloha-python)](https://github.com/LabNow-ai/aloha-python/blob/main/LICENSE) -[![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/LabNow-ai/aloha-python/build.yml?branch=main)](https://github.com/LabNow-ai/aloha-python/actions) -[![Join the Gitter Chat](https://img.shields.io/gitter/room/nwjs/nw.js.svg)](https://gitter.im/LabNow-ai/) -[![PyPI version](https://img.shields.io/pypi/v/aloha)](https://pypi.python.org/pypi/aloha/) -[![PyPI Downloads](https://img.shields.io/pypi/dm/aloha)](https://pepy.tech/badge/aloha/) -[![Code Activity](https://img.shields.io/github/commit-activity/m/LabNow-ai/aloha-python)](https://github.com/LabNow-ai/aloha-python/pulse) -[![Recent Code Update](https://img.shields.io/github/last-commit/LabNow-ai/aloha-python.svg)](https://github.com/LabNow-ai/aloha-python/stargazers) +你可以将 `aloha` 安装到现有 Python 项目中,也可以使用 `aloha-python` 仓库作为新微服务的 Docker 开发起点。 -如果这个项目对你有帮助,欢迎给我们点一个 STAR,或支持我们的开发工作![![GitHub Stars](https://img.shields.io/github/stars/LabNow-ai/aloha-python.svg?label=Stars&style=social)](https://github.com/LabNow-ai/aloha-python/stargazers) +[![PyPI 版本](https://img.shields.io/pypi/v/aloha)](https://pypi.org/project/aloha/) +[![构建状态](https://img.shields.io/github/actions/workflow/status/LabNow-ai/aloha-python/build.yml?branch=main)](https://github.com/LabNow-ai/aloha-python/actions) +[![许可证](https://img.shields.io/github/license/LabNow-ai/aloha-python)](https://github.com/LabNow-ai/aloha-python/blob/main/LICENSE) -`aloha` 是一个用于构建 Python 微服务的通用工具包,封装了常见组件与能力,例如: +## Aloha 提供什么 -- 快速创建 RESTful API 并启动服务 -- 日志工具 -- 环境、配置文件与资源文件管理 -- 连接常见数据库 -- 运行环境检测与监控 +- **HOCON 配置**:加载模块化配置文件,并通过环境变量选择运行配置 profile。 +- **面向服务的日志**:将 plain 或 JSON 日志输出到控制台和按日轮转的文件,并包含进程及主机元数据。 +- **数据库连接**:提供 PostgreSQL、MySQL、SQLite、DuckDB、MongoDB、Redis、Elasticsearch、Oracle 和 Kafka 等数据库或服务的操作工具。 +- **安全工具**:AES、RSA、JWT、哈希及密码保险库集成。 +- **Python Web 服务**:FastAPI/Uvicorn 应用辅助组件和可复用 HTTP 服务组件。 +- **测试与构建工具**:测试辅助工具,以及可选的 Cython 命令,用于将指定模块构建为本地扩展模块。 +- **容器化项目模板**:示例应用、Dockerfile、Docker Compose 开发环境和 CI/CD 脚本。 -## 安装 +## 安装 Python 包 -```title="安装 aloha 及扩展依赖" -pip install aloha[all] +Aloha 支持 Python 3.10 及以上版本。使用 pip 安装基础包: + +```bash +python -m pip install aloha +``` + +可选集成通过 extras 分组;只安装项目需要的依赖即可: + +```bash +python -m pip install "aloha[service]" # FastAPI 和 Uvicorn +python -m pip install "aloha[db]" # 数据库集成 +python -m pip install "aloha[all]" # 所有可选集成和工具 +``` + +更多 extras 和最小使用示例见[安装与快速开始](README-get-start.md)。 + +## 选择使用方式 + +### 在现有 Python 项目中使用 Aloha + +安装 Python 包,使用 HOCON 配置应用,然后导入所需模块: + +```python +from aloha.logger import LOG + +LOG.info("Service is ready") ``` -请注意,包名后的 `[all]` 表示额外依赖集合,用于启用更多能力。 +全局 logger 从 HOCON 的 `deploy` 配置读取日志级别、文件格式和控制台格式。详情参阅[配置指南](README-config.md)和[日志 API 参考](api/logging.md)。 + +### 从微服务模板开始 + +克隆仓库并启动开发容器: + +```bash +git clone https://github.com/LabNow-ai/aloha-python.git +cd aloha-python +./tool/cicd/run-dev.sh up +./tool/cicd/run-dev.sh enter +``` + +进入容器后运行示例服务和测试: + +```bash +cd /workspace/src +python3 main.py app_common.main +pytest ./ +``` + +`src/app_common/` 中的应用是可供改造的示例。完整步骤和目录说明参见[快速开始指南](README-get-start.md)。 + +## 文档导航 + +- [安装与快速开始](README-get-start.md) +- [配置与 HOCON profile](README-config.md) +- [CLI 命令](README-cli.md) +- [12-Factor 应用指南](README-12factor.md) +- [开发与 Docker 指南](README-develop.md) +- [API 参考](api/index.md) -可选扩展包括: +## 项目与社区 -- `all`:包含下面所有扩展 -- `service`:构建 RESTful API 所需依赖(`aloha` 基于 Tornado) -- `build`:将 Python 代码编译为二进制文件,便于源码保护 -- `db`:连接常见数据库,如 MySQL / PostgreSQL / Redis -- `stream`:基于 `confluent_kafka` 处理流式数据 -- `data`:使用 `pandas` 等库进行数据处理或数据科学任务 -- `report`:将数据或报告导出为 Excel 文件 -- `test`:单元测试工具 -- `docs`:文档构建工具 +- [GitHub 源码仓库](https://github.com/LabNow-ai/aloha-python) +- [PyPI Python 包](https://pypi.org/project/aloha/) +- [报告问题或提出功能建议](https://github.com/LabNow-ai/aloha-python/issues) diff --git a/notebook/test-api-query-postgresql.ipynb b/notebook/test-api-query-postgresql.ipynb index 7d5910f..5b63829 100644 --- a/notebook/test-api-query-postgresql.ipynb +++ b/notebook/test-api-query-postgresql.ipynb @@ -42,7 +42,6 @@ "outputs": [], "source": [ "from aloha.service.api.v0 import APICaller\n", - "\n", "from aloha.settings import SETTINGS\n", "\n", "api_environment = 'LOCAL' # DEV | STG | PRD\n", @@ -72,9 +71,9 @@ " resp = caller.call('/api_internal/query_postgres', timeout=(20, 2000), data={\"sql\": sql, \"config_profile\": config_profile, })\n", " data = resp['data']\n", " return pd.read_json(data)\n", - " except Exception as e:\n", + " except Exception:\n", " print(resp)\n", - " raise e" + " raise" ] }, { diff --git a/notebook/test-api-service.ipynb b/notebook/test-api-service.ipynb index 3fee31a..0a6c350 100644 --- a/notebook/test-api-service.ipynb +++ b/notebook/test-api-service.ipynb @@ -8,6 +8,7 @@ "outputs": [], "source": [ "import sys\n", + "\n", "sys.path.insert(0, '../src/')" ] }, diff --git a/notebook/test-db-postgres.ipynb b/notebook/test-db-postgres.ipynb index 1e75169..e3ddaa1 100644 --- a/notebook/test-db-postgres.ipynb +++ b/notebook/test-db-postgres.ipynb @@ -17,7 +17,9 @@ }, "outputs": [], "source": [ - "import os, sys\n", + "import os\n", + "import sys\n", + "\n", "sys.path.insert(0, '../src/')\n", "os.environ['DIR_RESOURCE'] = '../app/resource'" ] @@ -29,8 +31,8 @@ "outputs": [], "source": [ "import aloha\n", - "from aloha.settings import SETTINGS as S\n", "from aloha.db.postgres import PostgresOperator\n", + "from aloha.settings import SETTINGS as S\n", "\n", "print(aloha.__path__)" ] diff --git a/pkg/aloha/config/hocon.py b/pkg/aloha/config/hocon.py index 4221bb0..a61babf 100644 --- a/pkg/aloha/config/hocon.py +++ b/pkg/aloha/config/hocon.py @@ -72,11 +72,13 @@ def load_config_from_hocon_files(config_files: list, base_dir: str): :param base_dir: Base directory for resolving relative paths :return: Configuration as an AttrDict object """ - s = [] - for config_file in config_files: - f = f'include required("{config_file}")' - s.append(f) - f = "\n".join(s) - - config = ConfigFactory.parse_string(content=f, basedir=base_dir).as_plain_ordered_dict() + if len(config_files) == 0: + raise ValueError("Empty config files provided to load hocon config!") + elif len(config_files) == 1: + path_config = os.path.join(base_dir, config_files[0]) + config = ConfigFactory.parse_file(path_config).as_plain_ordered_dict() + else: + s = [f'include required("{config_file}")' for config_file in config_files] + f = "\n".join(s) + config = ConfigFactory.parse_string(content=f, basedir=base_dir).as_plain_ordered_dict() return AttrDict(config) diff --git a/pkg/aloha/db/duckdb.py b/pkg/aloha/db/duckdb.py index e119efb..936899c 100644 --- a/pkg/aloha/db/duckdb.py +++ b/pkg/aloha/db/duckdb.py @@ -1,5 +1,6 @@ """DuckDB connection helpers.""" +import re from pathlib import Path import duckdb @@ -89,17 +90,21 @@ def _initialize_schema(self): return try: + schema = self._config["schema"] + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", schema): + raise ValueError(f"Invalid schema name: {schema!r}") + schema_quoted = f'"{schema}"' if self._config["read_only"]: result = self.engine.connext().execute( text("SELECT schema_name FROM information_schema.schemata WHERE schema_name = :schema"), - {"schema": self._config["schema"]}, + {"schema": schema}, ) if not result.fetchone(): - raise RuntimeError(f"Schema '{self._config['schema']}' does not exist and read_only=True") + raise RuntimeError(f"Schema '{schema}' does not exist and read_only=True") else: - self.engine.connect().execute(text(f"CREATE SCHEMA IF NOT EXISTS {self._config['schema']}")) + self.engine.connect().exec_driver_sql(f"CREATE SCHEMA IF NOT EXISTS {schema_quoted}") - self.engine.connect().execute(text(f"SET schema '{self._config['schema']}'")) + self.engine.connect().exec_driver_sql(f"SET schema '{schema}'") except Exception as e: raise RuntimeError(f"Failed to initialize schema: {e}") from e diff --git a/pkg/aloha/logger/__init__.py b/pkg/aloha/logger/__init__.py index ade112a..7c90592 100644 --- a/pkg/aloha/logger/__init__.py +++ b/pkg/aloha/logger/__init__.py @@ -1,7 +1,9 @@ from ..settings import SETTINGS -from .logger import get_logger, getLogger +from .logger import get_formatter, get_logger, getLogger LOG = get_logger( level=SETTINGS.config.get("deploy", {}).get("log_level", 10), # 10 = logging.DEBUG + log_format_file=SETTINGS.config.get("deploy", {}).get("log_format_file", "json"), + log_format_stream=SETTINGS.config.get("deploy", {}).get("log_format_stream", "plain"), ) -__all__ = ("LOG", "getLogger", "get_logger") +__all__ = ("LOG", "getLogger", "get_formatter", "get_logger") diff --git a/pkg/aloha/logger/handler.py b/pkg/aloha/logger/handler.py index fd59779..d44c95c 100644 --- a/pkg/aloha/logger/handler.py +++ b/pkg/aloha/logger/handler.py @@ -10,9 +10,19 @@ class MultiProcessSafeDailyRotatingFileHandler(BaseRotatingHandler): - Utc not supported """ - def __init__(self, filename: str, encoding="utf8", delay=False, utc=False, **kwargs): + def __init__( + self, + filename: str, + encoding="utf8", + delay=False, + utc=False, + date_format="%Y-%m%d", + filename_suffix="", + **kwargs, + ): self.utc = utc - self.suffix = "%Y-%m%d" + self.date_format = date_format + self.filename_suffix = filename_suffix self.baseFilename = filename self.currentFileName = self._compute_fn() BaseRotatingHandler.__init__(self, filename, "a", encoding, delay) @@ -29,7 +39,13 @@ def doRollover(self): self.stream = self._open() def _compute_fn(self): - return self.baseFilename.replace(".log", "") + "_" + time.strftime(self.suffix, time.localtime()) + ".log" + return ( + self.baseFilename.removesuffix(".log") + + "_" + + time.strftime(self.date_format, time.localtime()) + + self.filename_suffix + + ".log" + ) def _open(self): return open(self.currentFileName, mode=self.mode, encoding=self.encoding) diff --git a/pkg/aloha/logger/logger.py b/pkg/aloha/logger/logger.py index 91af830..6a2d9c9 100644 --- a/pkg/aloha/logger/logger.py +++ b/pkg/aloha/logger/logger.py @@ -1,10 +1,176 @@ +import http +import json import logging import os import socket +import traceback +from copy import copy +from datetime import datetime from os.path import join as pjoin +from typing import TextIO from .handler import MultiProcessSafeDailyRotatingFileHandler +DEFAULT_LOG_FORMAT = "%(levelprefix)s %(asctime)s %(source)s %(message)s" +DEFAULT_ACCESS_LOG_FORMAT = "%(levelprefix)s %(message)s" +LEVEL_COLORS = { + 5: "34", + logging.DEBUG: "36", + logging.INFO: "32", + logging.WARNING: "33", + logging.ERROR: "31", + logging.CRITICAL: "91", +} +STATUS_COLORS = {1: "97", 2: "32", 3: "33", 4: "31", 5: "91"} +LOG_RECORD_FIELDS = frozenset(logging.LogRecord("", 0, "", 0, "", (), None).__dict__) | { + "message", + "asctime", + "levelprefix", + "source", +} + + +def _format_timestamp(timestamp: float) -> str: + return datetime.fromtimestamp(timestamp).astimezone().isoformat(timespec="milliseconds") + + +def _get_extra_fields(record: logging.LogRecord) -> dict: + return {key: value for key, value in record.__dict__.items() if key not in LOG_RECORD_FIELDS} + + +def _get_runtime_fields() -> dict: + return {"pid": os.getpid(), "hostname": socket.gethostname()} + + +def _format_access_message(args: tuple, use_colors: bool) -> str: + client_addr, method, full_path, http_version, status_code = args + try: + status_phrase = http.HTTPStatus(int(status_code)).phrase + except ValueError: + status_phrase = "" + + status = f"{status_code} {status_phrase}".rstrip() + request_line = f"{method} {full_path} HTTP/{http_version}" + if use_colors: + request_line = f"\x1b[1m{request_line}\x1b[0m" + color_status = STATUS_COLORS.get(int(status_code) // 100) + if color_status: + status = f"\x1b[{color_status}m{status}\x1b[0m" + return f'{client_addr} - "{request_line}" {status}' + + +def _get_access_fields(record: logging.LogRecord) -> dict: + fields_access = _get_runtime_fields() + fields_access.update( + { + key: value + for key, value in _get_extra_fields(record).items() + if key != "logger" and key not in fields_access + } + ) + return fields_access + + +class PlainFormatter(logging.Formatter): + """Format plain logs with an optional Uvicorn-style colored level prefix.""" + + def __init__(self, fmt: str, use_colors: bool = False, include_source: bool = True): + super().__init__(fmt) + self.use_colors = use_colors + self.include_source = include_source + + def formatMessage(self, record: logging.LogRecord) -> str: + record_copy = copy(record) + level_name = record_copy.levelname + if self.use_colors and record_copy.levelno in LEVEL_COLORS: + level_name = f"\x1b[{LEVEL_COLORS[record_copy.levelno]}m{level_name}\x1b[0m" + record_copy.__dict__["levelprefix"] = f"{level_name}:{' ' * (8 - len(record_copy.levelname))}" + if self.include_source: + record_copy.__dict__["source"] = f"{record_copy.filename}:{record_copy.lineno}" + else: + if isinstance(record_copy.args, tuple) and len(record_copy.args) == 5: + record_copy.msg = _format_access_message(record_copy.args, self.use_colors) + record_copy.args = () + fields_access = json.dumps(_get_access_fields(record_copy), ensure_ascii=False, default=str) + record_copy.msg = f"{record_copy.getMessage()} {fields_access}" + record_copy.args = () + record_copy.message = record_copy.msg + return super().formatMessage(record_copy) + + def formatTime(self, record: logging.LogRecord, datefmt: str | None = None) -> str: + return _format_timestamp(record.created) + + +class JsonFormatter(logging.Formatter): + """Format log records as JSON objects.""" + + def __init__(self, include_source: bool = True, include_logger: bool = True, include_extra: bool = False): + super().__init__() + self.include_source = include_source + self.include_logger = include_logger + self.include_extra = include_extra + + def format(self, record: logging.LogRecord) -> str: + payload = { + "timestamp": _format_timestamp(record.created), + "level": record.levelname, + **_get_runtime_fields(), + } + if self.include_logger: + payload["logger"] = record.name + payload["message"] = record.getMessage() + if self.include_source: + payload["source"] = f"{record.filename}:{record.lineno}" + if self.include_extra: + for key, value in _get_extra_fields(record).items(): + if key == "logger" and not self.include_logger: + continue + payload.setdefault(key, value) + if record.exc_info: + payload["exception"] = self.formatException(record.exc_info) + if record.stack_info and record.levelno > logging.INFO: + payload["stack_info"] = self.formatStack(record.stack_info) + return json.dumps(payload, ensure_ascii=False, default=str) + + +class StackInfoFilter(logging.Filter): + """Capture the calling stack for records above INFO when it was not supplied.""" + + def filter(self, record: logging.LogRecord) -> bool: + if record.levelno > logging.INFO and not record.stack_info: + frames = traceback.extract_stack() + callsite = next( + ( + index + for index in range(len(frames) - 1, -1, -1) + if frames[index].filename == record.pathname and frames[index].lineno == record.lineno + ), + None, + ) + if callsite is not None: + frames = frames[: callsite + 1] + else: + frames = frames[:-1] + record.stack_info = "Stack (most recent call last):\n" + "".join(traceback.format_list(frames)) + return True + + +def get_formatter( + log_format: str = "plain", + formatter_str: str | None = None, + use_colors: bool = False, + access_log: bool = False, +) -> logging.Formatter: + """Build a formatter from a predefined name or a custom format string.""" + if formatter_str: + return logging.Formatter(formatter_str) + if log_format == "plain": + format_log = DEFAULT_ACCESS_LOG_FORMAT if access_log else DEFAULT_LOG_FORMAT + return PlainFormatter(format_log, use_colors=use_colors, include_source=not access_log) + if log_format == "json": + return JsonFormatter(include_source=not access_log, include_logger=not access_log, include_extra=access_log) + raise ValueError(f"Unsupported log format: {log_format!r}. Choose 'plain' or 'json'.") + def setup_logger( logger: logging.Logger, @@ -12,6 +178,11 @@ def setup_logger( logger_name: str | None = None, module: str | None = None, formatter_str: str | None = None, + log_format: str | None = None, + log_format_file: str = "json", + log_format_stream: str = "plain", + stream: TextIO | None = None, + access_log: bool = False, ): """ Set up a logger with file and stream handlers. @@ -26,9 +197,16 @@ def setup_logger( :param logger_name: Name of the logger (optional) :param module: Module name for log file naming (optional) :param formatter_str: Custom log format string (optional) + :param log_format: Deprecated format override for both file and stream + :param log_format_file: Predefined format for the file handler + :param log_format_stream: Predefined format for the console handler + :param stream: Console output stream (defaults to stderr) + :param access_log: Use the access-log format and omit the source location """ if not logger.handlers: - formatter = logging.Formatter(formatter_str or "%(levelname)s> %(asctime)s> %(module)s:%(lineno)s> %(message)s") + if log_format is not None: + log_format_file = log_format + log_format_stream = log_format folder = os.environ.get("DIR_LOG", "logs") os.makedirs(folder, exist_ok=True) @@ -36,21 +214,30 @@ def setup_logger( if module is None: from ..settings import SETTINGS - module = SETTINGS.config.get("APP_MODULE") or os.environ.get("APP_MODULE", None) - - if logger_name is not None and len(logger_name) > 0: - logger_name = logger_name.strip().replace(" ", "_") + module = SETTINGS.config.get("APP_MODULE") or os.environ.get("APP_MODULE") + module = module or "default" - path_file = [module, logger_name, socket.gethostname(), f"p{os.getpid()}"] # module, logger_name, hostname, pid - path_file = "_".join(str(i) for i in path_file if i is not None and len(str(i)) > 0) - path_file = pjoin(folder, f"{path_file}.log") + path_file = pjoin(folder, f"{module}.log") - file_handler = MultiProcessSafeDailyRotatingFileHandler(path_file) - file_handler.setFormatter(formatter) + file_handler = MultiProcessSafeDailyRotatingFileHandler( + path_file, + filename_suffix=".access" if access_log else "", + ) + file_handler.setFormatter( + get_formatter(log_format=log_format_file, formatter_str=formatter_str, access_log=access_log) + ) + file_handler.addFilter(StackInfoFilter()) logger.addHandler(file_handler) - stream_handler = logging.StreamHandler() - stream_handler.setFormatter(formatter) + stream_handler = logging.StreamHandler(stream) + stream_formatter = get_formatter( + log_format=log_format_stream, + formatter_str=formatter_str, + use_colors=stream_handler.stream.isatty(), + access_log=access_log, + ) + stream_handler.setFormatter(stream_formatter) + stream_handler.addFilter(StackInfoFilter()) logger.addHandler(stream_handler) logger.setLevel(level) @@ -61,11 +248,11 @@ def get_logger(logger_name: str | None = None, level=logging.DEBUG, **kwargs) -> Get a configured logger instance. Creates or retrieves a logger by name and sets it up with file and stream handlers. - Accepts both string and integer log levels. + Accepts string and integer logging levels. If ``logger_name`` is ``None``, + configures the root logger. :param level: Logging level (int or str, default: DEBUG) - :param logger_name: Name of the logger (default: 'default') - :param args: Additional arguments passed to setup_logger + :param logger_name: Logger name (default: the root logger) :param kwargs: Additional keyword arguments passed to setup_logger :return: Configured logger instance """ diff --git a/pkg/aloha/service/api/v1.py b/pkg/aloha/service/api/v1.py index a4584ba..d4c2014 100644 --- a/pkg/aloha/service/api/v1.py +++ b/pkg/aloha/service/api/v1.py @@ -129,7 +129,7 @@ def wrap_request_data( """Wrap the payload with signature fields expected by v1 handlers.""" if app_id is None: app_id = next(iter(self.APP_ID_KEYS.keys())) - salt_uuid = salt_uuid or str(uuid.uuid1()) + salt_uuid = salt_uuid or str(uuid.uuid4()) sign = sign or sign_data( salt_uuid=salt_uuid, app_id=app_id, diff --git a/pkg/aloha/service/app.py b/pkg/aloha/service/app.py index 4523fae..2d1e94f 100644 --- a/pkg/aloha/service/app.py +++ b/pkg/aloha/service/app.py @@ -43,7 +43,8 @@ def start(self): host="0.0.0.0", port=port, workers=workers, - log_level="info", + log_config=None, + log_level=None, access_log=True, ) self._server = uvicorn.Server(config) diff --git a/pkg/aloha/service/web.py b/pkg/aloha/service/web.py index 057605d..d680cfd 100644 --- a/pkg/aloha/service/web.py +++ b/pkg/aloha/service/web.py @@ -4,20 +4,45 @@ import logging import os import re +import sys from typing import Any from fastapi import FastAPI, Request from fastapi.responses import JSONResponse, Response -from ..logger import LOG -from ..logger.logger import setup_logger +from ..logger import LOG, get_logger from ..settings import SETTINGS -setup_logger( - logging.getLogger("uvicorn.access"), - formatter_str="A> %(asctime)s> %(message)s", - module=f"access_{SETTINGS.config.get('APP_MODULE') or os.environ.get('APP_MODULE', 'default')}", +cfg_deploy = SETTINGS.config.get("deploy", {}) +level_log = cfg_deploy.get("log_level", logging.DEBUG) +format_file_log = cfg_deploy.get("log_format_file", "json") +format_stream_log = cfg_deploy.get("log_format_stream", "plain") +module_log = SETTINGS.config.get("APP_MODULE") or os.environ.get("APP_MODULE", "default") + +logger_uvicorn = logging.getLogger("uvicorn") +logger_uvicorn.setLevel(level_log) +for handler in logger_uvicorn.handlers[:]: + logger_uvicorn.removeHandler(handler) + handler.close() +logger_uvicorn.propagate = True + +logger_uvicorn_error = logging.getLogger("uvicorn.error") +logger_uvicorn_error.setLevel(level_log) +for handler in logger_uvicorn_error.handlers[:]: + logger_uvicorn_error.removeHandler(handler) + handler.close() +logger_uvicorn_error.propagate = True + +logger_uvicorn_access = get_logger( + "uvicorn.access", + level=level_log, + module=module_log, + log_format_file=format_file_log, + log_format_stream=format_stream_log, + stream=sys.stdout, + access_log=True, ) +logger_uvicorn_access.propagate = False def _load_routes(name: str) -> list[tuple[str, Any]]: @@ -80,8 +105,8 @@ def _setup_routes(self): self._register_handler(url, handler_class) s_log_msg = f"Loaded API module {url:<50}" if LOG.level < logging.INFO: - s_log_msg += f"\t from class {handler_class!s}" - LOG.info(s_log_msg) + s_log_msg += f" from class {handler_class!s}" + LOG.info(s_log_msg.strip()) def _register_handler(self, url: str, handler_class): """Register a handler class as FastAPI routes based on its methods.""" @@ -229,7 +254,7 @@ def _match_path(self, tornado_pattern: str, path: str) -> dict: def get_port(self) -> int: """Get the configured port.""" service_settings = self.config.get("service", {}) - port = service_settings.get("port") or int(os.environ.get("PORT_SVC", "8000")) + port = service_settings.get("port") or int(os.environ.get("PORT_SVC", "9000")) port = int(os.environ.get("PORT", port)) return port diff --git a/pkg/setup.py b/pkg/setup.py index e54f3ef..c28027b 100644 --- a/pkg/setup.py +++ b/pkg/setup.py @@ -1,6 +1,6 @@ import os import shutil -from datetime import datetime, timezone +from datetime import datetime from setuptools import setup @@ -17,7 +17,7 @@ # 2. Dynamic Version Generation # Writes the version to aloha/_version.py using the current timestamp. -_t = datetime.now(tz=timezone.utc) +_t = datetime.now().astimezone() _version = f"{_t.year}.{_t.month:02d}{_t.day:02d}.{_t.hour:02d}{_t.minute:02d}" version_file_path = os.path.join(base_dir, "aloha", "_version.py") diff --git a/src/README.md b/src/README.md index 3032011..b49125b 100644 --- a/src/README.md +++ b/src/README.md @@ -53,12 +53,10 @@ Run this command in your terminal: 3. **Container Start**: Docker Compose starts the container with the following features: - **Volume Mounts**: Your local code directories are mounted into the container, enabling live development (changes on your host are immediately visible in the container): - - `doc/` → `/root/doc` - - `notebook/` → `/root/notebook` - - `src/` → `/root/src` - - `app/` → `/root/app` + - Project root → `/workspace` - **Port Forwarding**: Exposes ports for your application and web interface - **Persistent Process**: The container runs `tail -f /dev/null` to stay active + - **Container User**: Runs as the host user's UID/GID so files written under `/workspace` remain accessible to the host user. ### Step 2: Enter the Development Container @@ -72,8 +70,8 @@ Once the environment is running, execute: - Uses `docker exec -it` to create an interactive terminal session - Attaches you to the running container with a bash shell -- You'll be logged in as the root user inside the container -- Your working directory will be `/root` +- The shell runs with your host user's UID/GID +- Your working directory will be `/workspace` **What you can do inside the container:** diff --git a/src/resource/config/deploy-DEV.conf b/src/resource/config/deploy-DEV.conf index b9f7fd1..9859134 100644 --- a/src/resource/config/deploy-DEV.conf +++ b/src/resource/config/deploy-DEV.conf @@ -1,5 +1,7 @@ deploy = { port_service = 9000 + log_format_file = "json" + log_format_stream = "plain" postgres_db0 = { "host": "localhost", diff --git a/src/resource/config/main.conf b/src/resource/config/main-DEV.conf similarity index 100% rename from src/resource/config/main.conf rename to src/resource/config/main-DEV.conf diff --git a/tool/cicd/docker-compose.app-demo.DEV.yml b/tool/cicd/docker-compose.app-demo.DEV.yml index 2e8cebf..b53d1aa 100644 --- a/tool/cicd/docker-compose.app-demo.DEV.yml +++ b/tool/cicd/docker-compose.app-demo.DEV.yml @@ -14,17 +14,21 @@ services: restart: unless-stopped # env_file: ["../credentials/DEV-app-demo.env"] environment: - # - PROFILE_ENV=${PROFILE_ENV:-DEV} + - PROFILE_ENV=${PROFILE_ENV:-DEV} - PROFILE_LOCALIZE=${PROFILE_LOCALIZE:-default} - - PYTHONPATH=/root/app/pkg:/root/app/src:/root/app/notebook - user: "0:0" - ports: [ "${PORT_APP:-9000}:9000", "${PORT_WEB:-3000}:3000" ] + - TZ=Asia/Shanghai + - PYTHONPATH=/workspace/pkg:/workspace/src:/workspace/notebook + - HOME=/home/${CONTAINER_UID:-0} + user: "${CONTAINER_UID:-0}:${CONTAINER_GID:-0}" + ports: ["${PORT_APP:-9000}:9000", "${PORT_WEB:-3000}:3000"] + working_dir: /workspace volumes: - - ../../:/root/app:rw + - ../../:/workspace:rw + # SSH keys and config for git + - "$HOME/.ssh:/home/${CONTAINER_UID:-0}/.ssh:rw" # AI4Coding tools: - - "$HOME/.claude:/root/.claude:rw" - - "$HOME/.codex:/root/.codex:rw" - - "$HOME/.copilot:/root/.copilot:rw" - - "$HOME/.gemini:/root/.gemini:rw" - working_dir: /root/app + - "$HOME/.claude:/home/${CONTAINER_UID:-0}/.claude:rw" + - "$HOME/.codex:/home/${CONTAINER_UID:-0}/.codex:rw" + - "$HOME/.copilot:/home/${CONTAINER_UID:-0}/.copilot:rw" + - "$HOME/.gemini:/home/${CONTAINER_UID:-0}/.gemini:rw" command: "tail -f /dev/null" diff --git a/tool/cicd/run-dev.sh b/tool/cicd/run-dev.sh index 3905528..1654a73 100755 --- a/tool/cicd/run-dev.sh +++ b/tool/cicd/run-dev.sh @@ -5,10 +5,13 @@ ACTION="${1:-up}" USERNAME="$(whoami)" USERID="$(id -u)" +GROUPID="$(id -g)" BASE_APP_PORT=30000 BASE_WEB_PORT=33000 +export CONTAINER_UID="$USERID" +export CONTAINER_GID="$GROUPID" export PORT_APP=$((BASE_APP_PORT + USERID)) export PORT_WEB=$((BASE_WEB_PORT + USERID)) export PROJECT_NAME="dev-app-demo-${USERNAME}" @@ -43,7 +46,7 @@ require_ports_free() { # ---------- Action Dispatcher ---------- echo "----------------------------------------" -echo "User: $USERNAME (UID: $USERID)" +echo "User: $USERNAME (UID: $USERID, GID: $GROUPID)" echo "Project Name: $PROJECT_NAME" echo "Container: $CONTAINER_NAME" echo "App Port Expose: $PORT_APP"