diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..aaa444a --- /dev/null +++ b/.dockerignore @@ -0,0 +1,14 @@ +.git +.github +backups/ +screenshots/ +venv/ +__pycache__/ +*.pyc +*.pyo +*.pyd +playwright_cookies.json +*.egg-info/ +dist/ +build/ +.env diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml new file mode 100644 index 0000000..b940e53 --- /dev/null +++ b/.github/workflows/black.yml @@ -0,0 +1,11 @@ +name: Lint with Black + +on: [push, pull_request] + +jobs: + black: + timeout-minutes: 10 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: psf/black@stable diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml new file mode 100644 index 0000000..7dc3345 --- /dev/null +++ b/.github/workflows/docker-image.yml @@ -0,0 +1,33 @@ +name: Build, upload, and publish Docker images + +on: + release: + types: + - created + pull_request: + branches: + - master + +jobs: + docker: + timeout-minutes: 20 + + runs-on: ubuntu-latest + + permissions: + # See https://docs.pypi.org/trusted-publishers/using-a-publisher/ + id-token: write + contents: write + + steps: + - uses: actions/checkout@v6 + + - name: Build image + run: docker build -t jira_backup . + + - name: Test image + run: docker run --rm jira_backup --help + + # TODO: Upload Docker image to the release. + + # TODO: Add tags and publish Docker image to the public Docker registry. diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml new file mode 100644 index 0000000..6f401dc --- /dev/null +++ b/.github/workflows/mypy.yml @@ -0,0 +1,26 @@ +name: Lint with MyPy + +on: [push, pull_request] + +jobs: + mypy: + timeout-minutes: 10 + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r mypy-requirements.txt .[all] + + - name: Run MyPy + run: | + mypy . diff --git a/.github/workflows/python-wheel.yml b/.github/workflows/python-wheel.yml new file mode 100644 index 0000000..344ac8b --- /dev/null +++ b/.github/workflows/python-wheel.yml @@ -0,0 +1,51 @@ +name: Build, upload, and publish Python wheels + +on: + release: + types: + - created + pull_request: + branches: + - master + +jobs: + wheel: + timeout-minutes: 20 + + runs-on: ubuntu-latest + + permissions: + # See https://docs.pypi.org/trusted-publishers/using-a-publisher/ + id-token: write + contents: write + + steps: + - uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine build + + - name: Build wheel + run: python -m build + + - name: Upload to the release + if: github.event_name == 'release' + uses: softprops/action-gh-release@v2 + with: + files: dist/*.whl + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Publish to the public PyPI + if: github.event_name == 'release' + env: + TWINE_USERNAME: ${{ secrets.PUBLIC_PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PUBLIC_PYPI_PASSWORD }} + run: twine upload dist/*.whl diff --git a/.gitignore b/.gitignore index 7723488..804f3e6 100644 --- a/.gitignore +++ b/.gitignore @@ -21,10 +21,22 @@ var/ _config.json venv/ +.venv/ .vscode/ node_modules/ package-lock.json test.py backups/* !backups/.gitkeep -wizard.pyc +*.pyc +__pycache__/ +.pytest_cache/ +.mypy_cache/ +.coverage +coverage.xml + +# IDEs +.idea/ + +# Files that may contain secrets +config.yaml diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..81525f3 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,8 @@ +FROM python:3.14-alpine +COPY ./ /backup +RUN adduser -D backup +RUN chown -R backup:backup /backup +WORKDIR /backup +RUN pip install --no-cache-dir ".[all]" +USER backup +ENTRYPOINT ["python", "-m", "jira_backup"] diff --git a/README.md b/README.md index bd24e5d..37445ba 100644 --- a/README.md +++ b/README.md @@ -1,65 +1,245 @@ +# Jira Backup Python + [![datree-badge](https://s3.amazonaws.com/catalog.static.datree.io/datree-badge-28px.svg)](https://datree.io/?src=badge) -# Introduction -Jira and Confluence are not (officially) supporting the option of creating automatic backups for their cloud instance. -This project was created to provide a fully automated infrastructure for backing up Atlassian Cloud Jira or Confluence instances on a periodic basis. - -There are shell and bash scripts out there, which were created in order to download the backup file locally without the use of the "backup manager" UI, -but most of them are not maintained and throwing errors. So, this project is aiming for full backup automation, and therefore this is the features road map: - -:white_check_mark: Create a script in python -:white_check_mark: Support creating config.json from user input ('wizard') -:white_check_mark: Download backup file locally -:white_check_mark: Add an option to stream backup file to S3 -:white_check_mark: Check how to manually create a cron task on OS X / Linux -:white_check_mark: Check how to manually create a schedule task on windows -:black_square_button: Support adding cron / scheduled task from script     - -# Installation -## Prerequisite: -:heavy_plus_sign: python 2.7.x or python 3.x.x -:heavy_plus_sign: [virtualenv](https://pypi.org/project/virtualenv/) installed globally (pip install virtualenv) - -## Instructions: -1. Create and start [virtual environment](https://python-guide-cn.readthedocs.io/en/latest/dev/virtualenvs.html) (in this example, the virtualenv will be called "venv") -2. Install requirements +[![Python 3.8+](https://img.shields.io/badge/python-3.8+-blue.svg)](https://www.python.org/downloads/) +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) + +A Python-based backup tool for Atlassian Cloud Jira and Confluence instances with multi-cloud storage support and automated scheduling. + +## 🚀 Features + +- **Jira & Confluence Backups**: Create backups for both Jira and Confluence Cloud instances +- **Multi-Cloud Support**: Stream backups directly to AWS S3, Google Cloud Storage, or Azure Blob Storage +- **Local Download**: Option to download backup files locally +- **Cross-Platform Scheduling**: Automatically create cron jobs (Linux/macOS) or scheduled tasks (Windows) +- **Configuration Wizard**: Interactive setup for easy configuration +- **API Token Authentication**: Secure authentication using Atlassian API tokens + +## 📋 Prerequisites + +- Python 3.8 or higher +- Atlassian Cloud account (Jira and/or Confluence) +- API token from [Atlassian](https://id.atlassian.com/manage/api-tokens) +- (Optional) Cloud storage account: AWS, Google Cloud, or Azure + +## 🛠️ Installation + +### From PyPI + +```shell +pip install jira_backup +``` + +### From source + +1. **Clone the repository** + ```bash + git clone https://github.com/datreeio/jira-backup-py.git + cd jira-backup-py + ``` + +2. **Create a virtual environment** + ```bash + python -m venv venv + source venv/bin/activate # On Windows: venv\Scripts\activate + ``` + +3. **Install this project with its dependencies** + ```bash + pip install . + ``` + +### Post-installation steps + +1. Go to [Atlassian API Tokens](https://id.atlassian.com/manage/api-tokens) and create a token. +2. See the [Configuration](#configuration) section below. + +## ⚙️ Configuration + +### Configuration file + +- `jira_backup` looks for a `config.yaml` file in the working directory. +- You can pass a config file explicitly with `-C /path/to/config.yaml` or `--config /path/to/config.yaml`. +- You can [generate a config file using the wizard](#configuration-wizard). +- For source checkouts, you can start from the example file: `cp config.example.yaml config.yaml` + +#### Configuration validation + +- The config file is loaded with `yaml.safe_load` and validated with Pydantic before any backup starts. +- Keys are case-insensitive. +- Unknown keys are rejected. +- The YAML loader will discard duplicate keys. Only the last occurrence is kept. +- Booleans must be real YAML booleans (`true` or `false`), not quoted strings. +- `host_url` must be a valid Atlassian hostname without a scheme or other URL + components. + +#### Configuration example + +```yaml +--- +host_url: "your-instance.atlassian.net" +user_email: "your.email@company.com" +api_token: "your-api-token" +include_attachments: false +download_locally: true + +# AWS S3 Configuration (optional) +upload_to_s3: + aws_endpoint_url: "" + aws_region: "us-east-1" + s3_bucket: "my-backup-bucket" + s3_dir: "Atlassian/" + aws_access_key: "your-access-key" + aws_secret_key: "your-secret-key" + aws_is_secure: true + +# Google Cloud Storage Configuration (optional) +upload_to_gcp: + gcp_project_id: "my-project-id" + gcs_bucket: "my-backup-bucket" + gcs_dir: "Atlassian/" + gcp_service_account_key: "/path/to/service-account-key.json" + +# Azure Blob Storage Configuration (optional) +upload_to_azure: + azure_account_name: "mystorageaccount" + azure_container: "my-backup-container" + azure_dir: "Atlassian/" + azure_connection_string: "DefaultEndpointsProtocol=https;AccountName=..." + azure_account_key: "" + +# Custom Filename (optional) +# Supports placeholders: +# - {timestamp} - Current timestamp (DDMMYYYY_HHMM) +# - {date} - Current date (YYYY-MM-DD) +# - {time} - Current time (HHMM) +# - {uuid} - UUID from backup URL +# - {type} - Backup type (jira/confluence) +custom_filename: + jira: "jira.{timestamp}" + confluence: "confluence.{timestamp}" +``` + +#### Configuration wizard + +For interactive setup, run: +```bash +python -m jira_backup -w +``` + +This will guide you through setting up basic Jira credentials and S3 configuration. + +## 🚀 Usage + +### Manual Backup + +```bash +# Backup Jira (default) +python -m jira_backup -j + +# Backup Confluence +python -m jira_backup -c + +# Run configuration wizard +python -m jira_backup -w +``` + +### Automated Scheduling + +Set up scheduled backups using system schedulers: + +```bash +# Setup automated Jira backup every 4 days at 10:00 AM (default) +python -m jira_backup -s + +# Setup automated Confluence backup every 7 days at 2:30 PM +python -m jira_backup -s --schedule-days 7 --schedule-time 14:30 --schedule-service confluence + +# Setup automated Jira backup every 2 days at 6:00 AM +python -m jira_backup -s --schedule-days 2 --schedule-time 06:00 --schedule-service jira ``` -$(venv) pip install -r requirements.txt -``` -3. Generate an API token at https://id.atlassian.com/manage/api-tokens -![Screenshot](https://github.com/datreeio/jira-backup-py/blob/master/screenshots/atlassian-api-token.png) -4. Fill the details at the [config.yaml file](https://github.com/datreeio/jira-backup-py/blob/master/config.json) or run the backup.py script with '-w' flag -5. Run backup.py script with the flag '-j' to backup Jira or '-c' to backup Confluence + +This will create: +- **Linux/macOS**: A cron job in your crontab +- **Windows**: A scheduled task in Task Scheduler + +Scheduled tasks store an absolute config path. +If you do not pass `-C` or `--config`, +the scheduler uses `config.yaml` from the directory where you ran the scheduling command. + +### Command Line Options + +| Option | Description | +|----------------------|---------------------------------------------------------------------------| +| `-j, --jira` | Backup Jira (default if no service specified) | +| `-c, --confluence` | Backup Confluence | +| `-C, --config` | Path to the config file (default: `config.yaml` in the current directory) | +| `-w, --wizard` | Run configuration wizard | +| `-s, --schedule` | Setup automated scheduled backup | +| `--schedule-days` | Frequency in days for scheduled backup (default: 4) | +| `--schedule-time` | Time for scheduled backup in HH:MM format (default: 10:00) | +| `--schedule-service` | Service for scheduled backup (jira/confluence, default: jira) | + +## 🔧 Advanced Configuration + +### Minimal Configuration + +If you only want to download backups locally without cloud storage, +simply omit the `upload_to_xxx` sections: + +```yaml +--- +host_url: "your-instance.atlassian.net" +user_email: "your.email@company.com" +api_token: "your-api-token" +include_attachments: false +download_locally: true ``` -$(venv) python backup.py -``` -![Screenshot](https://github.com/datreeio/jira-backup-py/blob/master/screenshots/terminal.png) - -## What's next? -It depends on your needs. I, for example, use this script together with [serverless](https://serverless.com/) to create a periodic [AWS lambda](https://aws.amazon.com/lambda/) which triggered every 4 days, creating a backup and upload it directly to S3. - -There is a more "stupid" option to get the same result - by creating a cron / scheduled task on your local machine: -* **OS X / Linux:** set a cron task with crontab -``` -echo "* * * * * cd %script dir% && %activate virtualenv% && python backup.py > %log name% 2>&1" | crontab - -``` -Example for adding a cron task which will run every 4 days, at 10:00 + +### Multiple Cloud Providers + +You can configure multiple cloud storage providers simultaneously - the script will upload to all configured destinations: + +```yaml +upload_to_s3: + s3_bucket: "my-s3-bucket" + # ... S3 config + +upload_to_gcp: + gcs_bucket: "my-gcs-bucket" + # ... GCP config + +upload_to_azure: + azure_container: "my-azure-container" + # ... Azure config ``` -echo "0 10 */4 * * cd ~/Dev/jira-backup-py && source venv/bin/activate && python backup.py > backup_script.log 2>&1" | crontab - -``` - -* **Windows:** set a scheduled task with task scheduler -``` -schtasks /create /tn "%task name%" /sc DAILY /mo %number of days% /tr "%full path to win_task_wrapper.bat%" /st %start time% -``` -Example for adding a scheduled task which will run every 4 days, at 10:00 -``` -schtasks /create /tn "jira-backup" /sc DAILY /mo 4 /tr "C:\jira-backup-py\win_task_wrapper.bat" /st 10:00 -``` -# Changelog: -* 04 SEP 2020 - Support Confluence backup -* 16 JAN 2019 - Updated script to work w/ [API token](https://confluence.atlassian.com/cloud/api-tokens-938839638.html), instead personal Jira user name and password - -# Resources: -:heavy_plus_sign: [JIRA support - How to Automate Backups for JIRA Cloud applications](https://confluence.atlassian.com/jirakb/how-to-automate-backups-for-jira-cloud-applications-779160659.html) -:heavy_plus_sign: [Atlassian Labs' automatic-cloud-backup script](https://bitbucket.org/atlassianlabs/automatic-cloud-backup/src/d43ca5f33192e78b2e1869ab7c708bb32bfd7197/backup.ps1?at=master&fileviewer=file-view-default) -:heavy_plus_sign: [A more maintainable version of Atlassian Labs' script](https://github.com/mattock/automatic-cloud-backup) + +## 🤝 Contributing + +Contributions are welcome! Please feel free to submit issues and pull requests. + +## 📝 Changelog + +- **2025-06-24**: Added separate cron schedules for Jira and Confluence backups +- **2025-06-24**: Made cloud storage configuration sections optional +- **2025-06-24**: Added automated scheduling support for backup tasks +- **2025-06-23**: Added Google Cloud Storage and Azure Blob Storage support +- **2020-09-04**: Added Confluence backup support +- **2019-01-16**: Updated to use API tokens instead of passwords + +## 📜 License + +This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. + +## 🙏 Acknowledgments + +- Original concept inspired by [Atlassian Labs' automatic-cloud-backup](https://bitbucket.org/atlassianlabs/automatic-cloud-backup/) +- Thanks to all contributors who have helped improve this project + +## 📞 Support + +- **Issues**: [GitHub Issues](https://github.com/yourusername/jira-backup-py/issues) + +--- + +**Note**: This tool is not officially supported by Atlassian. Use at your own risk and always verify your backups are working correctly. diff --git a/backup.py b/backup.py deleted file mode 100644 index a43add1..0000000 --- a/backup.py +++ /dev/null @@ -1,125 +0,0 @@ -import json -import yaml -import time -import os -import argparse -import requests -import boto -from boto.s3.key import Key -import wizard -from time import gmtime, strftime - - -def read_config(): - config_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'config.yaml') - with open(config_path, 'r') as config_file: - return yaml.full_load(config_file) - - -class Atlassian: - def __init__(self, config): - self.config = config - self.session = requests.Session() - self.session.auth = (config['USER_EMAIL'], config['API_TOKEN']) - self.session.headers.update({'Content-Type': 'application/json', 'Accept': 'application/json'}) - self.payload = {"cbAttachments": self.config['INCLUDE_ATTACHMENTS'], "exportToCloud": "true"} - self.start_confluence_backup = 'https://{}/wiki/rest/obm/1.0/runbackup'.format(self.config['HOST_URL']) - self.start_jira_backup = 'https://{}/rest/backup/1/export/runbackup'.format(self.config['HOST_URL']) - self.backup_status = {} - self.wait = 10 - - def create_confluence_backup(self): - backup = self.session.post(self.start_confluence_backup, data=json.dumps(self.payload)) - if backup.status_code != 200: - raise Exception(backup, backup.text) - else: - print('-> Backup process successfully started') - confluence_backup_status = 'https://{}/wiki/rest/obm/1.0/getprogress'.format(self.config['HOST_URL']) - time.sleep(self.wait) - while 'fileName' not in self.backup_status.keys(): - self.backup_status = json.loads(self.session.get(confluence_backup_status).text) - print('Current status: {progress}; {description}'.format( - progress=self.backup_status['alternativePercentage'], - description=self.backup_status['currentStatus'])) - time.sleep(self.wait) - return 'https://{url}/wiki/download/{file_name}'.format( - url=self.config['HOST_URL'], file_name=self.backup_status['fileName']) - - def create_jira_backup(self): - backup = self.session.post(self.start_jira_backup, data=json.dumps(self.payload)) - if backup.status_code != 200: - raise Exception(backup, backup.text) - else: - task_id = json.loads(backup.text)['taskId'] - print('-> Backup process successfully started: taskId={}'.format(task_id)) - jira_backup_status = 'https://{jira_host}/rest/backup/1/export/getProgress?taskId={task_id}'.format( - jira_host=self.config['HOST_URL'], task_id=task_id) - time.sleep(self.wait) - while 'result' not in self.backup_status.keys(): - self.backup_status = json.loads(self.session.get(jira_backup_status).text) - print('Current status: {status} {progress}; {description}'.format( - status=self.backup_status['status'], - progress=self.backup_status['progress'], - description=self.backup_status['description'])) - time.sleep(self.wait) - return '{prefix}/{result_id}'.format( - prefix='https://' + self.config['HOST_URL'] + '/plugins/servlet', result_id=self.backup_status['result']) - - def download_file(self, url, local_filename): - print('-> Downloading file from URL: {}'.format(url)) - r = self.session.get(url, stream=True) - file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'backups', local_filename) - with open(file_path, 'wb') as file_: - for chunk in r.iter_content(chunk_size=1024): - if chunk: - file_.write(chunk) - print(file_path) - - def stream_to_s3(self, url, remote_filename): - print('-> Streaming to S3') - - if self.config['UPLOAD_TO_S3']['AWS_ACCESS_KEY'] == '': - connect = boto.connect_s3() - else: - connect = boto.connect_s3( - aws_access_key_id=self.config['UPLOAD_TO_S3']['AWS_ACCESS_KEY'], - aws_secret_access_key=self.config['UPLOAD_TO_S3']['AWS_SECRET_KEY'] - ) - - bucket = connect.get_bucket(self.config['UPLOAD_TO_S3']['S3_BUCKET']) - r = self.session.get(url, stream=True) - if r.status_code == 200: - k = Key(bucket) - k.key = remote_filename - k.content_type = r.headers['content-type'] - k.set_contents_from_string(r.content) - return - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('-w', action='store_true', dest='wizard', help='activate config wizard') - parser.add_argument('-c', action='store_true', dest='confluence', help='activate confluence backup') - parser.add_argument('-j', action='store_true', dest='jira', help='activate jira backup') - # print('debug command-line: {}'.format(parser.parse_args())) - if parser.parse_args().wizard: - wizard.create_config() - config = read_config() - - if config['HOST_URL'] == 'something.atlassian.net': - raise ValueError('You forgated to edit config.json or to run the backup script with "-w" flag') - - print('-> Starting backup; include attachments: {}'.format(config['INCLUDE_ATTACHMENTS'])) - atlass = Atlassian(config) - if parser.parse_args().confluence: backup_url = atlass.create_confluence_backup() - else: backup_url = atlass.create_jira_backup() - - print('-> Backup URL: {}'.format(backup_url)) - file_name = '{timestemp}_{uuid}.zip'.format( - timestemp=time.strftime('%d%m%Y_%H%M'), uuid=backup_url.split('/')[-1].replace('?fileId=', '')) - - if config['DOWNLOAD_LOCALLY'] == 'true': - atlass.download_file(backup_url, file_name) - - if config['UPLOAD_TO_S3']['S3_BUCKET'] != '': - atlass.stream_to_s3(backup_url, file_name) \ No newline at end of file diff --git a/__init__.py b/ci-constraints.txt similarity index 100% rename from __init__.py rename to ci-constraints.txt diff --git a/config.example.yaml b/config.example.yaml new file mode 100644 index 0000000..cff0522 --- /dev/null +++ b/config.example.yaml @@ -0,0 +1,36 @@ +--- +host_url: "something.atlassian.net" +user_email: "user.name@company.com" +api_token: "replace-with-atlassian-api-token" +include_attachments: false +download_locally: true + +# Optional. To skip S3 upload, leave `s3_bucket` empty or remove the `upload_to_s3` section. +upload_to_s3: + aws_endpoint_url: "" + aws_region: "us-east-1" + s3_bucket: "" + s3_dir: "Atlassian/" + aws_access_key: "" + aws_secret_key: "" + aws_is_secure: true + +# Optional. To skip GCS upload, leave `gcs_bucket` empty or remove the `upload_to_gcp` section. +upload_to_gcp: + gcp_project_id: "" + gcs_bucket: "" + gcs_dir: "Atlassian/" + gcp_service_account_key: null + +# Optional. To skip Azure upload, leave `azure_container` empty or remove the `upload_to_azure` section. +upload_to_azure: + azure_account_name: "" + azure_container: "" + azure_dir: "Atlassian/" + azure_connection_string: "" + azure_account_key: "" + +# Optional. Remove this section to use the default {timestamp}_{uuid}.zip name. +custom_filename: + jira: "jira.{timestamp}" + confluence: "confluence.{timestamp}" diff --git a/config.yaml b/config.yaml deleted file mode 100644 index 961fa7a..0000000 --- a/config.yaml +++ /dev/null @@ -1,10 +0,0 @@ ---- -HOST_URL: "something.atlassian.net" -USER_EMAIL: "email address for the Atlassian account you're using to create the token (user.name@company.com)" -API_TOKEN: "token ID generated at https://id.atlassian.com/manage/api-tokens" -INCLUDE_ATTACHMENTS: "include attachments? this will make the backup size bigger - true / false" -DOWNLOAD_LOCALLY: "download the backup file to backups folder? true / false" -UPLOAD_TO_S3: - S3_BUCKET: "S3 bucket name (empty value will skip this step)" - AWS_ACCESS_KEY: "not mandatory if already set on the machine with AWS CLI" - AWS_SECRET_KEY: "not mandatory if already set on the machine with AWS CLI" \ No newline at end of file diff --git a/mypy-requirements.txt b/mypy-requirements.txt new file mode 100644 index 0000000..3a2689b --- /dev/null +++ b/mypy-requirements.txt @@ -0,0 +1,6 @@ +mypy==2.1.0 +boto3-stubs==1.* +types_boto3==1.* +types_pyyaml==6.* +types_requests==2.* +types_setuptools>=80 diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 0000000..df1e6e8 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,3 @@ +[mypy] +strict = true +exclude = ^(build|venv|tmp|cache|dist)/ diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..fe0ceb8 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,70 @@ +[build-system] +requires = ["setuptools>=80", "setuptools-scm[toml]>=8", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "jira_backup" +dynamic = ["version"] +description = "Backup Jira and Confluence Cloud data to local storage or cloud object stores." +readme = { file = "README.md", content-type = "text/markdown" } +license = "MIT" +license-files = ["LICENSE"] +authors = [] +requires-python = ">=3.8" +dependencies = [ + "pydantic>=2,<3", + "PyYAML>=6.0.2,<7", + "requests>=2.32.3,<3", +] +keywords = ["jira", "confluence", "backup", "atlassian", "s3", "gcs", "azure"] +classifiers = [ + "Development Status :: 4 - Beta", + "Environment :: Console", + "Intended Audience :: System Administrators", + "Operating System :: Microsoft :: Windows", + "Operating System :: MacOS", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: System :: Archiving :: Backup", + "Topic :: Utilities", +] + +[project.optional-dependencies] +s3 = [ + "boto3>=1.35.26,<2", +] +gcp = [ + "google-cloud-storage>=2.18.0,<3", +] +azure = [ + "azure-storage-blob>=12.22.0,<13", +] +all = [ + "boto3>=1.35.26,<2", + "google-cloud-storage>=2.18.0,<3", + "azure-storage-blob>=12.22.0,<13", +] + +[project.scripts] +jira-backup = "jira_backup:main" + +[project.urls] +Homepage = "https://github.com/datreeio/jira-backup-py" +Repository = "https://github.com/datreeio/jira-backup-py" +Issues = "https://github.com/datreeio/jira-backup-py/issues" + +[tool.setuptools] +package-dir = {"" = "src"} + +[tool.setuptools.packages.find] +where = ["src"] +include = ["jira_backup*"] + +[tool.setuptools_scm] +fallback_version = "v0.0.1-dev" diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index cbe0492..0000000 --- a/requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -boto==2.48.0 -certifi==2018.4.16 -chardet==3.0.4 -idna==2.6 -requests==2.20.0 -urllib3==1.24.2 -PyYAML==5.3.1 \ No newline at end of file diff --git a/src/jira_backup/__init__.py b/src/jira_backup/__init__.py new file mode 100644 index 0000000..5096a16 --- /dev/null +++ b/src/jira_backup/__init__.py @@ -0,0 +1,8 @@ +from ._backup import Atlassian +from ._config import Config, read_config + +__all__ = [ + "Atlassian", + "Config", + "read_config", +] diff --git a/src/jira_backup/__main__.py b/src/jira_backup/__main__.py new file mode 100644 index 0000000..9d8ffca --- /dev/null +++ b/src/jira_backup/__main__.py @@ -0,0 +1,4 @@ +from ._backup import main + +if __name__ == "__main__": + main() diff --git a/src/jira_backup/_backup.py b/src/jira_backup/_backup.py new file mode 100644 index 0000000..122eff9 --- /dev/null +++ b/src/jira_backup/_backup.py @@ -0,0 +1,626 @@ +import argparse +import json +import os +import platform +import shlex +import subprocess +import sys +import time +from importlib import import_module +from pathlib import Path +from typing import Dict, Any, Literal + +import requests +import urllib3 + +from ._config import read_config, Config + + +class OptionalExtraMissingError(RuntimeError): + pass + + +def import_optional_extra(module_name: str, extra_name: str, purpose: str) -> Any: + try: + return import_module(module_name) + except ModuleNotFoundError as e: + missing_root = module_name.split(".")[0] + if getattr(e, "name", None) in (missing_root, module_name): + raise OptionalExtraMissingError( + f"{purpose} requires the optional '{extra_name}' extra. " + f'Install it with: pip install "jira_backup[{extra_name}]"' + ) from e + raise + + +def ensure_upload_extras(config: Config) -> None: + if config.upload_to_s3 and config.upload_to_s3.s3_bucket: + import_optional_extra("boto3", "s3", "S3 uploads") + + if config.upload_to_gcp and config.upload_to_gcp.gcs_bucket: + import_optional_extra("google.cloud.storage", "gcp", "GCS uploads") + + if config.upload_to_azure and config.upload_to_azure.azure_container: + import_optional_extra( + "azure.storage.blob", "azure", "Azure Blob Storage uploads" + ) + + +class Atlassian: + def __init__(self, config: Config) -> None: + self.config = config + self.session = requests.Session() + self.session.auth = (config.user_email, config.api_token) + self.session.headers.update( + {"Content-Type": "application/json", "Accept": "application/json"} + ) + self.payload = { + "cbAttachments": self.config.include_attachments, + "exportToCloud": "true", + } + self.start_confluence_backup = "https://{}/wiki/rest/obm/1.0/runbackup".format( + self.config.host_url + ) + self.start_jira_backup = "https://{}/rest/backup/1/export/runbackup".format( + self.config.host_url + ) + self.get_last_jira_backup = "https://{}/rest/backup/1/export/lastTaskId".format( + self.config.host_url + ) + self.backup_status: Dict[str, Any] = {} + self.wait = 10 + + def generate_filename(self, backup_url: str, backup_type: str = "jira") -> str: + """ + Generate filename based on config or default pattern. + Supports placeholders: + - {timestamp} - Current timestamp in format DDMMYYYY_HHMM + - {date} - Current date in format YYYY-MM-DD + - {time} - Current time in format HHMM + - {uuid} - UUID from backup URL + - {type} - Backup type (jira or confluence) + """ + uuid = backup_url.split("/")[-1].replace("?fileId=", "") + timestamp = time.strftime("%d%m%Y_%H%M") + + custom_pattern = self.config.custom_filename + + if custom_pattern is None: + pattern = "" + elif backup_type == "confluence": + pattern = custom_pattern.confluence + else: + pattern = custom_pattern.jira + + if pattern: + filename = pattern.format( + timestamp=timestamp, + date=time.strftime("%Y-%m-%d"), + time=time.strftime("%H%M"), + uuid=uuid, + type=backup_type, + ) + if not filename.endswith(".zip"): + filename += ".zip" + return filename + else: + return "{timestamp}_{uuid}.zip".format(timestamp=timestamp, uuid=uuid) + + def create_confluence_backup(self) -> str: + backup = self.session.post( + self.start_confluence_backup, data=json.dumps(self.payload) + ) + + if backup.status_code not in (200, 406): + raise Exception(backup, backup.text) + + print("-> Backup process successfully started") + confluence_backup_status = "https://{}/wiki/rest/obm/1.0/getprogress".format( + self.config.host_url + ) + time.sleep(self.wait) + while "fileName" not in self.backup_status.keys(): + self.backup_status = json.loads( + self.session.get(confluence_backup_status).text + ) + print( + "Current status: {progress}; {description}".format( + progress=self.backup_status["alternativePercentage"], + description=self.backup_status["currentStatus"], + ) + ) + time.sleep(self.wait) + return "https://{url}/wiki/download/{file_name}".format( + url=self.config.host_url, file_name=self.backup_status["fileName"] + ) + + def create_jira_backup(self) -> str: + backup = self.session.post( + self.start_jira_backup, data=json.dumps(self.payload) + ) + task_id = "" + + if backup.status_code == 412: + print("-> Backup already exists. Atlassian said: {}".format(backup.text)) + backup = self.session.get(self.get_last_jira_backup) + if backup.status_code == 200: + print("-> Downloading existing backup: taskId={}".format(task_id)) + task_id = backup.text + else: + raise Exception(backup, backup.text) + + elif backup.status_code == 200: + task_id = json.loads(backup.text)["taskId"] + print("-> Backup process successfully started: taskId={}".format(task_id)) + else: + raise Exception(backup, backup.text) + + jira_backup_status = "https://{jira_host}/rest/backup/1/export/getProgress?taskId={task_id}".format( + jira_host=self.config.host_url, task_id=task_id + ) + time.sleep(self.wait) + while "result" not in self.backup_status.keys(): + self.backup_status = json.loads(self.session.get(jira_backup_status).text) + print( + "Current status: {status} {progress}; {description}".format( + status=self.backup_status["status"], + progress=self.backup_status["progress"], + description=self.backup_status["description"], + ) + ) + time.sleep(self.wait) + return "{prefix}/{result_id}".format( + prefix="https://" + self.config.host_url + "/plugins/servlet", + result_id=self.backup_status["result"], + ) + + def download_file(self, url: str, local_filename: str, max_retries: int = 5) -> str: + print("-> Downloading file from URL: {}".format(url)) + file_path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "backups", local_filename + ) + + # check if alredy downloaded partially + downloaded_bytes = 0 + if os.path.exists(file_path): + downloaded_bytes = os.path.getsize(file_path) + print("-> Resuming download from byte {}".format(downloaded_bytes)) + + for attempt in range(max_retries): + try: + headers = {} + if downloaded_bytes > 0: + headers["Range"] = f"bytes={downloaded_bytes}-" + + r = self.session.get(url, stream=True, headers=headers, timeout=60) + + # get complete size + if "content-range" in r.headers: + total_size = int(r.headers["content-range"].split("/")[-1]) + elif "content-length" in r.headers: + total_size = int(r.headers["content-length"]) + downloaded_bytes + else: + total_size = 0 + + mode = "ab" if downloaded_bytes > 0 else "wb" + + with open(file_path, mode) as file_: + for chunk in r.iter_content(chunk_size=1024 * 1024): # 1MB chunks + if chunk: + file_.write(chunk) + downloaded_bytes += len(chunk) + + # show progress + if total_size > 0: + percent = (downloaded_bytes / total_size) * 100 + downloaded_gb = downloaded_bytes / (1024**3) + total_gb = total_size / (1024**3) + print( + f"\r-> Progress: {percent:.1f}% ({downloaded_gb:.2f} GB / {total_gb:.2f} GB)", + end="", + flush=True, + ) + + print("\n-> Download completed: {}".format(file_path)) + return file_path + + except ( + requests.exceptions.RequestException, + urllib3.exceptions.ProtocolError, + ) as e: + print(f"\n-> Download interrupted: {e}") + print(f"-> Retry {attempt + 1}/{max_retries} in 10 seconds...") + time.sleep(10) + + # refresh downloaded_bytes for resume + if os.path.exists(file_path): + downloaded_bytes = os.path.getsize(file_path) + + raise Exception(f"Download failed after {max_retries} retries") + + def stream_to_s3(self, url: str, remote_filename: str) -> None: + print("-> Streaming to S3") + boto3 = import_optional_extra("boto3", "s3", "S3 uploads") + upload_config = self.config.upload_to_s3 + + if upload_config is None: + raise ValueError( + "S3 upload was requested but upload_to_s3 is not configured" + ) + + if upload_config.aws_access_key == "": + s3_client = boto3.client("s3") + else: + s3_client = boto3.client( + "s3", + aws_access_key_id=upload_config.aws_access_key, + aws_secret_access_key=upload_config.aws_secret_key, + region_name=upload_config.aws_region or None, + endpoint_url=upload_config.aws_endpoint_url or None, + use_ssl=upload_config.aws_is_secure, + ) + + bucket_name = upload_config.s3_bucket + r = self.session.get(url, stream=True) + if r.status_code == 200: + key = "{s3_bucket}{s3_filename}".format( + s3_bucket=upload_config.s3_dir, + s3_filename=remote_filename, + ) + + s3_client.upload_fileobj( + r.raw, + Bucket=bucket_name, + Key=key, + ExtraArgs={"ContentType": r.headers["content-type"]}, + ) + + def stream_to_gcs(self, url: str, remote_filename: str) -> None: + print("-> Streaming to GCS") + storage = import_optional_extra("google.cloud.storage", "gcp", "GCS uploads") + upload_config = self.config.upload_to_gcp + + if upload_config is None: + raise ValueError( + "GCS upload was requested but upload_to_gcp is not configured" + ) + + if upload_config.gcp_service_account_key: + client = storage.Client.from_service_account_json( + upload_config.gcp_service_account_key, + project=upload_config.gcp_project_id, + ) + else: + client = storage.Client(project=upload_config.gcp_project_id) + + bucket_name = upload_config.gcs_bucket + bucket = client.bucket(bucket_name) + + r = self.session.get(url, stream=True) + if r.status_code == 200: + blob_name = "{gcs_dir}{filename}".format( + gcs_dir=upload_config.gcs_dir, + filename=remote_filename, + ) + + blob = bucket.blob(blob_name) + blob.content_type = r.headers.get("content-type", "application/zip") + + blob.upload_from_file(r.raw, content_type=blob.content_type) + + def stream_to_azure(self, url: str, remote_filename: str) -> None: + print("-> Streaming to Azure Blob Storage") + blob_module = import_optional_extra( + "azure.storage.blob", "azure", "Azure Blob Storage uploads" + ) + blob_service_client_class = blob_module.BlobServiceClient + upload_config = self.config.upload_to_azure + + if upload_config is None: + raise ValueError( + "Azure upload was requested but upload_to_azure is not configured" + ) + + if upload_config.azure_connection_string: + blob_service_client = blob_service_client_class.from_connection_string( + upload_config.azure_connection_string + ) + else: + account_url = ( + f"https://{upload_config.azure_account_name}.blob.core.windows.net" + ) + blob_service_client = blob_service_client_class( + account_url=account_url, + credential=upload_config.azure_account_key, + ) + + container_name = upload_config.azure_container + + r = self.session.get(url, stream=True) + if r.status_code == 200: + blob_name = "{azure_dir}{filename}".format( + azure_dir=upload_config.azure_dir, + filename=remote_filename, + ) + + blob_client = blob_service_client.get_blob_client( + container=container_name, blob=blob_name + ) + + blob_client.upload_blob( + r.raw, + content_type=r.headers.get("content-type", "application/zip"), + overwrite=True, + ) + + +def setup_scheduled_task( + *, + frequency_days: int = 4, + time_hour: int = 10, + time_minute: int = 0, + service_type: Literal["jira", "confluence"] = "jira", + config_path: Path, +) -> bool: + system = platform.system().lower() + + if system in ["linux", "darwin"]: + return setup_cron_task( + frequency_days=frequency_days, + time_hour=time_hour, + time_minute=time_minute, + service_type=service_type, + config_path=config_path, + ) + elif system == "windows": + return setup_windows_task( + frequency_days=frequency_days, + time_hour=time_hour, + time_minute=time_minute, + service_type=service_type, + config_path=config_path, + ) + else: + raise Exception(f"Unsupported operating system: {system}") + + +def setup_cron_task( + *, + frequency_days: int, + time_hour: int, + time_minute: int, + service_type: Literal["jira", "confluence"], + config_path: Path, +) -> bool: + service_flag = "-j" if service_type == "jira" else "-c" + backup_command = shlex.join( + [ + sys.executable, + "-m", + "jira_backup", + service_flag, + "-C", + config_path.as_posix(), + ] + ) + cron_command = f"{time_minute} {time_hour} */{frequency_days} * * {backup_command}" + + try: + result = subprocess.run(["crontab", "-l"], capture_output=True, text=True) + existing_cron = result.stdout if result.returncode == 0 else "" + + # Remove only the cron entry for the same service type + lines = existing_cron.strip().split("\n") if existing_cron.strip() else [] + updated_lines = [] + skip_next = False + + for i, line in enumerate(lines): + if skip_next: + skip_next = False + continue + + # Check if this is a comment line for jira-backup-py + if ( + "jira-backup-py automated backup" in line + and f"({service_type})" in line + ): + # Check if the next line contains the cron command for this service + if i + 1 < len(lines) and service_flag in lines[i + 1]: + skip_next = True # Skip both the comment and the command + print(f"-> Updating existing {service_type} backup schedule...") + continue + + updated_lines.append(line) + + existing_cron = "\n".join(updated_lines) + "\n" if updated_lines else "" + new_cron = ( + existing_cron + + f"# jira-backup-py automated backup ({service_type})\n{cron_command}\n" + ) + + process = subprocess.Popen(["crontab", "-"], stdin=subprocess.PIPE, text=True) + process.communicate(input=new_cron) + + if process.returncode == 0: + print( + f"-> Successfully scheduled {service_type} backup to run every {frequency_days} days at {time_hour:02d}:{time_minute:02d}" + ) + return True + else: + print("-> Failed to create cron job") + return False + + except Exception as e: + print(f"-> Error setting up cron job: {e}") + return False + + +def setup_windows_task( + *, + frequency_days: int, + time_hour: int, + time_minute: int, + service_type: Literal["jira", "confluence"], + config_path: Path, +) -> bool: + task_name = f"jira-backup-py-{service_type}" + service_flag = "-j" if service_type == "jira" else "-c" + backup_command = subprocess.list2cmdline( + [sys.executable, "-m", "jira_backup", service_flag, "-C", config_path] + ) + cmd = [ + "schtasks", + "/create", + "/tn", + task_name, + "/sc", + "DAILY", + "/mo", + str(frequency_days), + "/tr", + backup_command, + "/st", + f"{time_hour:02d}:{time_minute:02d}", + "/f", + ] + + try: + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode == 0: + print( + f"-> Successfully scheduled {service_type} backup to run every {frequency_days} days at {time_hour:02d}:{time_minute:02d}" + ) + return True + else: + print(f"-> Failed to create scheduled task: {result.stderr}") + return False + except Exception as e: + print(f"-> Error setting up scheduled task: {e}") + return False + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "-C", + "--config", + type=str, + dest="config_file", + default="config.yaml", + help="path to config file", + ) + parser.add_argument( + "-w", action="store_true", dest="wizard", help="activate config wizard" + ) + parser.add_argument( + "-c", action="store_true", dest="confluence", help="activate confluence backup" + ) + parser.add_argument( + "-j", action="store_true", dest="jira", help="activate jira backup" + ) + parser.add_argument( + "-s", + "--schedule", + action="store_true", + dest="schedule", + help="setup automated scheduled backup", + ) + parser.add_argument( + "--schedule-days", + type=int, + default=4, + help="frequency in days for scheduled backup (default: 4)", + ) + parser.add_argument( + "--schedule-time", + type=str, + default="10:00", + help="time for scheduled backup in HH:MM format (default: 10:00)", + ) + parser.add_argument( + "--schedule-service", + type=str, + choices=["jira", "confluence"], + default="jira", + help="service type for scheduled backup (default: jira)", + ) + args = parser.parse_args() + config_path = Path(args.config_file) + + if args.wizard: + from ._wizard import create_config + + create_config(config_path=config_path) + + if args.schedule: + try: + time_parts = args.schedule_time.split(":") + hour = int(time_parts[0]) + minute = int(time_parts[1]) if len(time_parts) > 1 else 0 + + if not (0 <= hour <= 23) or not (0 <= minute <= 59): + raise ValueError("Invalid time format") + + if not config_path.exists(): + print("-> Error: Can't schedule script without a config file.") + exit(1) + + setup_scheduled_task( + frequency_days=args.schedule_days, + time_hour=hour, + time_minute=minute, + service_type=args.schedule_service, + config_path=config_path.resolve(), + ) + print("-> Scheduled task setup completed") + exit(0) + except ValueError as e: + print(f"-> Error: Invalid time format. Use HH:MM format (e.g., 10:30)") + exit(1) + except Exception as e: + print(f"-> Error setting up scheduled task: {e}") + exit(1) + + try: + config = read_config(config_path=config_path) + except Exception as e: + print(f"-> Error: {e}", file=sys.stderr) + exit(1) + + if config.host_url == "something.atlassian.net": + print('-> Error: You forgot to edit config.yaml or to run the backup script with "-w" flag', file=sys.stderr) + exit(1) + + try: + ensure_upload_extras(config) + except OptionalExtraMissingError as e: + print(f"-> Error: {e}", file=sys.stderr) + exit(1) + + print( + "-> Starting backup; include attachments: {}".format(config.include_attachments) + ) + + atlass = Atlassian(config) + + backup_type = "confluence" if args.confluence else "jira" + if args.confluence: + backup_url = atlass.create_confluence_backup() + else: + backup_url = atlass.create_jira_backup() + + print("-> Backup URL: {}".format(backup_url)) + file_name = atlass.generate_filename(backup_url, backup_type) + print("-> Generated filename: {}".format(file_name)) + + if config.download_locally: + atlass.download_file(backup_url, file_name) + + if config.upload_to_s3 and config.upload_to_s3.s3_bucket: + atlass.stream_to_s3(backup_url, file_name) + + if config.upload_to_gcp and config.upload_to_gcp.gcs_bucket: + atlass.stream_to_gcs(backup_url, file_name) + + if config.upload_to_azure and config.upload_to_azure.azure_container: + atlass.stream_to_azure(backup_url, file_name) diff --git a/src/jira_backup/_config.py b/src/jira_backup/_config.py new file mode 100644 index 0000000..890b5cf --- /dev/null +++ b/src/jira_backup/_config.py @@ -0,0 +1,215 @@ +from __future__ import annotations + +from collections.abc import Mapping +from pathlib import Path +from urllib.parse import urlsplit + +import yaml +from pydantic import ( + AliasChoices, + BaseModel, + ConfigDict, + Field, + StrictBool, + ValidationError, + field_validator, + model_validator, +) + + +def _is_valid_hostname(value: str) -> bool: + try: + parsed = urlsplit(f"//{value}") + hostname = parsed.hostname + port = parsed.port + except ValueError: + return False + + if ( + hostname is None + or parsed.netloc != value + or parsed.username is not None + or parsed.password is not None + or port is not None + or parsed.path + or parsed.query + or parsed.fragment + ): + return False + + try: + ascii_hostname = hostname.encode("idna").decode("ascii") + except UnicodeError: + return False + + if len(ascii_hostname) > 253: + return False + + labels = ascii_hostname.split(".") + return all( + 1 <= len(label) <= 63 + and label[0].isalnum() + and label[-1].isalnum() + and all(character.isalnum() or character == "-" for character in label) + for label in labels + ) + + +class ConfigModel(BaseModel): + model_config = ConfigDict(extra="forbid", populate_by_name=True) + + @model_validator(mode="before") + @classmethod + def normalize_case_insensitive_keys(cls, data: object) -> object: + if not isinstance(data, Mapping): + return data + + key_map = cls.case_insensitive_key_map() + normalized: dict[str, object] = {} + original_keys: dict[str, str] = {} + + for key, value in data.items(): + if not isinstance(key, str): + raise ValueError("config keys must be strings") + + normalized_key = key_map.get(key.casefold(), key) + if normalized_key in normalized: + original_key = original_keys[normalized_key] + raise ValueError( + f"duplicate config keys {original_key!r} and {key!r} " + f"both map to {normalized_key!r}" + ) + + normalized[normalized_key] = value + original_keys[normalized_key] = key + + return normalized + + @classmethod + def case_insensitive_key_map(cls) -> dict[str, str]: + key_map: dict[str, str] = {} + + for field_name, field_info in cls.model_fields.items(): + key_map[field_name.casefold()] = field_name + validation_alias = field_info.validation_alias + + if isinstance(validation_alias, str): + key_map[validation_alias.casefold()] = field_name + continue + + if isinstance(validation_alias, AliasChoices): + for alias in validation_alias.choices: + if isinstance(alias, str): + key_map[alias.casefold()] = field_name + + return key_map + + +class ConfigUploadToS3(ConfigModel): + aws_endpoint_url: str = "" + aws_region: str = Field( + default="", + validation_alias=AliasChoices("aws_region", "aws_s3_region"), + ) + s3_bucket: str = "" + s3_dir: str = "" + aws_access_key: str = Field( + default="", + validation_alias=AliasChoices("aws_access_key", "aws_access_key_id"), + ) + aws_secret_key: str = Field( + default="", + validation_alias=AliasChoices("aws_secret_key", "aws_secret_access_key"), + ) + aws_is_secure: StrictBool = True + + +class ConfigUploadToGCP(ConfigModel): + gcp_project_id: str = "" + gcs_bucket: str = "" + gcs_dir: str = "" + gcp_service_account_key: str | None = None + + +class ConfigUploadToAzure(ConfigModel): + azure_account_name: str = "" + azure_container: str = "" + azure_dir: str = "" + azure_connection_string: str = "" + azure_account_key: str = "" + + +class ConfigCustomFilename(ConfigModel): + confluence: str = "" + jira: str = "" + + +class Config(ConfigModel): + host_url: str + user_email: str + api_token: str + include_attachments: StrictBool + download_locally: StrictBool + upload_to_s3: ConfigUploadToS3 | None = None + upload_to_gcp: ConfigUploadToGCP | None = None + upload_to_azure: ConfigUploadToAzure | None = None + custom_filename: ConfigCustomFilename | None = None + + @field_validator("user_email", "api_token") + @classmethod + def required_string_must_not_be_blank(cls, value: str) -> str: + value = value.strip() + if not value: + raise ValueError("must not be blank") + return value + + @field_validator("host_url") + @classmethod + def host_url_must_be_hostname(cls, value: str) -> str: + if not value.strip(): + raise ValueError("must not be blank") + + if not _is_valid_hostname(value): + raise ValueError( + "must be a valid hostname without a scheme or other URL components" + ) + return value + + +def _format_validation_errors(error: ValidationError) -> str: + formatted_errors: list[str] = [] + + for details in error.errors(include_input=False): + location = ".".join(str(part) for part in details["loc"]) or "" + formatted_errors.append( + f"{location}: {details['msg']} [type={details['type']}]" + ) + + return "\n".join(formatted_errors) + + +def read_config(*, config_path: Path) -> Config: + try: + with config_path.open("r", encoding="utf-8") as config_file: + config_data = yaml.safe_load(config_file) + except yaml.YAMLError as e: + raise ValueError(f"Invalid YAML in config file {config_path}: {e}") from e + except FileNotFoundError as e: + raise FileNotFoundError( + f"Config file not found: {config_path}. " + "Copy config.example.yaml to config.yaml or pass -C." + ) from e + + if config_data is None: + config_data = {} + + if not isinstance(config_data, dict): + raise ValueError(f"Config file {config_path} must contain a YAML mapping.") + + try: + return Config.model_validate(config_data) + except ValidationError as e: + validation_errors = _format_validation_errors(e) + raise ValueError( + f"Invalid config file {config_path}:\n{validation_errors}" + ) from e diff --git a/src/jira_backup/_wizard.py b/src/jira_backup/_wizard.py new file mode 100644 index 0000000..eff0e57 --- /dev/null +++ b/src/jira_backup/_wizard.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +import sys +from collections.abc import Callable +from pathlib import Path + +import yaml +from pydantic import ValidationError + +from ._config import Config, ConfigUploadToS3, _format_validation_errors + + +def _input_config() -> Config: + prompts: dict[str, Callable[[], object]] = { + "host_url": lambda: input("What is your Jira host name? "), + "user_email": lambda: input("What is your Jira account email address? "), + "api_token": lambda: input("Paste your Jira API token: "), + "include_attachments": lambda: input_boolean( + "Do you want to include attachments?" + ), + "download_locally": lambda: input_boolean( + "Do you want to download the backup file locally?" + ), + } + values = {field_name: prompt() for field_name, prompt in prompts.items()} + + while True: + try: + return Config.model_validate(values) + except ValidationError as error: + print( + f"-> Invalid configuration:\n{_format_validation_errors(error)}", + file=sys.stderr, + ) + invalid_fields = { + details["loc"][0] + for details in error.errors(include_input=False) + if details["loc"] and isinstance(details["loc"][0], str) + } + retry_fields = { + field_name for field_name in invalid_fields if field_name in prompts + } + if not retry_fields: + raise + + for field_name, prompt in prompts.items(): + if field_name in retry_fields: + values[field_name] = prompt() + + +def create_config(*, config_path: Path) -> None: + custom_config = _input_config() + + if input_boolean("Do you want to upload the backup file to S3?"): + s3_config = ConfigUploadToS3( + aws_endpoint_url=input("What is your AWS endpoint url? "), + aws_region=input("What is your AWS region? "), + s3_bucket=input("What is the S3 bucket name? "), + s3_dir=input("What is the S3 directory for upload? (example Atlassian/) "), + aws_access_key=input("What is your AWS access key? "), + aws_secret_key=input("What is your AWS secret key? "), + aws_is_secure=input_boolean("Do you want to use SSL?"), + ) + custom_config = Config.model_validate( + {**custom_config.model_dump(), "upload_to_s3": s3_config} + ) + + config_path.parent.mkdir(parents=True, exist_ok=True) + with config_path.open("w", encoding="utf-8") as config_file: + yaml.safe_dump( + custom_config.model_dump(exclude_none=True), + config_file, + default_flow_style=False, + sort_keys=False, + ) + + print(f"-> Wrote configuration to {config_path.resolve()}") + + +def parse_boolean(s: str) -> bool: + return s.lower() in ("yes", "true", "t", "1", "y") + + +def input_boolean(q: str) -> bool: + return parse_boolean(input(f"{q} (y/n) ")) diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..61f1088 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + +from jira_backup._config import Config, read_config + + +def make_config(*, host_url: str) -> Config: + return Config.model_validate( + { + "host_url": host_url, + "user_email": "backup@example.com", + "api_token": "token", + "include_attachments": True, + "download_locally": True, + } + ) + + +class ConfigTests(unittest.TestCase): + def test_host_url_accepts_bare_hostname(self) -> None: + for hostname in ( + "example.atlassian.net", + "EXAMPLE.atlassian.net", + "jira-backup.internal", + "localhost", + ): + with self.subTest(hostname=hostname): + self.assertEqual(make_config(host_url=hostname).host_url, hostname) + + def test_host_url_rejects_url_components_and_whitespace(self) -> None: + for value in ( + "http://example.atlassian.net", + "HTTPS://example.atlassian.net", + "example.atlassian.net/path", + "example.atlassian.net?path=wrong", + "example.atlassian.net#fragment", + "user@example.atlassian.net", + "example.atlassian.net@evil.example", + "example.atlassian.net:443", + " example.atlassian.net", + "example.atlassian.net ", + "example .atlassian.net", + ): + with self.subTest(value=value): + with self.assertRaises(ValueError): + make_config(host_url=value) + + def test_host_url_rejects_invalid_hostname_syntax(self) -> None: + for value in ( + "-example.atlassian.net", + "example-.atlassian.net", + "example..atlassian.net", + "example_atlassian.net", + ): + with self.subTest(value=value): + with self.assertRaises(ValueError): + make_config(host_url=value) + + +class ReadConfigTests(unittest.TestCase): + def test_validation_errors_omit_rejected_credential_values(self) -> None: + api_token = "super-secret-api-token" + aws_secret_key = "super-secret-aws-key" + + with tempfile.TemporaryDirectory() as temp_dir: + config_path = Path(temp_dir) / "config.yaml" + config_path.write_text( + "\n".join( + [ + "host_url: example.atlassian.net", + "user_email: backup@example.com", + f"api_tokn: {api_token}", + "include_attachments: true", + "download_locally: true", + "upload_to_s3:", + f" aws_secret_keey: {aws_secret_key}", + ] + ), + encoding="utf-8", + ) + + with self.assertRaises(ValueError) as raised: + read_config(config_path=config_path) + + message = str(raised.exception) + self.assertIn("api_token: Field required [type=missing]", message) + self.assertIn( + "api_tokn: Extra inputs are not permitted [type=extra_forbidden]", + message, + ) + self.assertIn( + "upload_to_s3.aws_secret_keey: " + "Extra inputs are not permitted [type=extra_forbidden]", + message, + ) + self.assertNotIn(api_token, message) + self.assertNotIn(aws_secret_key, message) + self.assertNotIn("input_value", message) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_wizard.py b/tests/test_wizard.py new file mode 100644 index 0000000..b66b191 --- /dev/null +++ b/tests/test_wizard.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +import io +import sys +import tempfile +import unittest +from contextlib import redirect_stderr, redirect_stdout +from pathlib import Path +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + +from jira_backup._config import read_config +from jira_backup._wizard import create_config + + +class CreateConfigTests(unittest.TestCase): + def test_invalid_values_show_errors_and_reprompt_only_invalid_fields(self) -> None: + answers = [ + "https://example.atlassian.net", + "", + "", + "y", + "n", + "example.atlassian.net", + "backup@example.com", + "api-token", + "n", + ] + + with tempfile.TemporaryDirectory() as temp_dir: + config_path = Path(temp_dir) / "config.yaml" + with patch("builtins.input", side_effect=answers) as input_mock: + with redirect_stderr(io.StringIO()) as stderr: + with redirect_stdout(io.StringIO()): + create_config(config_path=config_path) + + config = read_config(config_path=config_path) + + self.assertEqual(input_mock.call_count, len(answers)) + self.assertEqual(config.host_url, "example.atlassian.net") + self.assertEqual(config.user_email, "backup@example.com") + self.assertEqual(config.api_token, "api-token") + self.assertTrue(config.include_attachments) + self.assertFalse(config.download_locally) + + error_output = stderr.getvalue() + self.assertIn("-> Invalid configuration:", error_output) + self.assertIn("host_url: Value error", error_output) + self.assertIn("user_email: Value error, must not be blank", error_output) + self.assertIn("api_token: Value error, must not be blank", error_output) + self.assertNotIn("Traceback", error_output) + + +if __name__ == "__main__": + unittest.main() diff --git a/win_task_wrapper.bat b/win_task_wrapper.bat index 3df372e..4cdf91d 100644 --- a/win_task_wrapper.bat +++ b/win_task_wrapper.bat @@ -7,12 +7,12 @@ popd cd %script_dir% REM print output and error to file and then to console: -REM powershell -window minimized -Command "& venv\Scripts\activate.bat; python backup.py 2>&1 | tee backup_script.log" +REM powershell -window minimized -Command "& venv\Scripts\activate.bat; python -m jira_backup 2>&1 | tee backup_script.log" REM print output and error to console: -powershell -Command "& venv\Scripts\activate.bat; python backup.py" +powershell -Command "& venv\Scripts\activate.bat; python -m jira_backup" REM print output and error to file: -REM powershell -window minimized -Command "& venv\Scripts\activate.bat; python backup.py >> backup_script.log 2>&1" +REM powershell -window minimized -Command "& venv\Scripts\activate.bat; python -m jira_backup >> backup_script.log 2>&1" pause diff --git a/wizard.py b/wizard.py deleted file mode 100644 index 5b4780d..0000000 --- a/wizard.py +++ /dev/null @@ -1,31 +0,0 @@ -import os -import json - - -def create_config(): - jira_host = raw_input("What is your Jira host name? ") - user = raw_input("What is your Jira account email address? ") - password = raw_input("Paste your Jira API token: ") - attachments = raw_input("Do you want to include attachments? (true / false) ") - download_locally = raw_input("Do you want to download the backup file locally? (true / false) ") - custom_config = { - 'JIRA_HOST': jira_host, - 'INCLUDE_ATTACHMENTS': attachments.lower(), - 'JIRA_EMAIL': user, - 'API_TOKEN': password, - 'DOWNLOAD_LOCALLY': download_locally.lower(), - 'UPLOAD_TO_S3': { - 'S3_BUCKET': "", - 'AWS_ACCESS_KEY': "", - 'AWS_SECRET_KEY': "" - } - } - upload_backup = raw_input("Do you want to upload the backup file to S3? (true / false) ") - if upload_backup.lower() == 'true': - custom_config['UPLOAD_TO_S3']['S3_BUCKET'] = raw_input("What is the S3 bucket name? ") - custom_config['UPLOAD_TO_S3']['AWS_ACCESS_KEY'] = raw_input("What is your AWS access key? ") - custom_config['UPLOAD_TO_S3']['AWS_SECRET_KEY'] = raw_input("What is your AWS secret key? ") - - config_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'config.json') - with open(config_path, 'w+') as config_file: - json.dump(custom_config, config_file)