diff --git a/app/__init__.py b/app/__init__.py index 946668d..b20f44e 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -185,14 +185,12 @@ def create_ui_app(test_config: Optional[Dict[str, Any]] = None) -> Flask: def _configure_cors(app: Flask) -> None: origins = app.config.get("CORS_ORIGINS", ["*"]) - methods = app.config.get("CORS_METHODS", ["GET", "PUT", "POST", "DELETE", "OPTIONS"]) - allow_headers = app.config.get( - "CORS_ALLOW_HEADERS", - ["Content-Type", "X-Access-Key", "X-Secret-Key", "X-Amz-Date", "X-Amz-SignedHeaders"], - ) + methods = app.config.get("CORS_METHODS", ["GET", "PUT", "POST", "DELETE", "OPTIONS", "HEAD"]) + allow_headers = app.config.get("CORS_ALLOW_HEADERS", ["*"]) + expose_headers = app.config.get("CORS_EXPOSE_HEADERS", ["*"]) CORS( app, - resources={r"/*": {"origins": origins, "methods": methods, "allow_headers": allow_headers}}, + resources={r"/*": {"origins": origins, "methods": methods, "allow_headers": allow_headers, "expose_headers": expose_headers}}, supports_credentials=True, ) diff --git a/app/config.py b/app/config.py index 47afc96..ef9420d 100644 --- a/app/config.py +++ b/app/config.py @@ -59,6 +59,7 @@ class AppConfig: cors_origins: list[str] cors_methods: list[str] cors_allow_headers: list[str] + cors_expose_headers: list[str] session_lifetime_days: int auth_max_attempts: int auth_lockout_minutes: int @@ -148,18 +149,9 @@ class AppConfig: return parts or default cors_origins = _csv(str(_get("CORS_ORIGINS", "*")), ["*"]) - cors_methods = _csv(str(_get("CORS_METHODS", "GET,PUT,POST,DELETE,OPTIONS")), ["GET", "PUT", "POST", "DELETE", "OPTIONS"]) - cors_allow_headers = _csv(str(_get("CORS_ALLOW_HEADERS", "Content-Type,X-Access-Key,X-Secret-Key,X-Amz-Algorithm,X-Amz-Credential,X-Amz-Date,X-Amz-Expires,X-Amz-SignedHeaders,X-Amz-Signature")), [ - "Content-Type", - "X-Access-Key", - "X-Secret-Key", - "X-Amz-Algorithm", - "X-Amz-Credential", - "X-Amz-Date", - "X-Amz-Expires", - "X-Amz-SignedHeaders", - "X-Amz-Signature", - ]) + cors_methods = _csv(str(_get("CORS_METHODS", "GET,PUT,POST,DELETE,OPTIONS,HEAD")), ["GET", "PUT", "POST", "DELETE", "OPTIONS", "HEAD"]) + cors_allow_headers = _csv(str(_get("CORS_ALLOW_HEADERS", "*")), ["*"]) + cors_expose_headers = _csv(str(_get("CORS_EXPOSE_HEADERS", "*")), ["*"]) session_lifetime_days = int(_get("SESSION_LIFETIME_DAYS", 30)) bucket_stats_cache_ttl = int(_get("BUCKET_STATS_CACHE_TTL", 60)) # Default 60 seconds @@ -191,6 +183,7 @@ class AppConfig: cors_origins=cors_origins, cors_methods=cors_methods, cors_allow_headers=cors_allow_headers, + cors_expose_headers=cors_expose_headers, session_lifetime_days=session_lifetime_days, auth_max_attempts=auth_max_attempts, auth_lockout_minutes=auth_lockout_minutes, @@ -234,6 +227,7 @@ class AppConfig: "CORS_ORIGINS": self.cors_origins, "CORS_METHODS": self.cors_methods, "CORS_ALLOW_HEADERS": self.cors_allow_headers, + "CORS_EXPOSE_HEADERS": self.cors_expose_headers, "SESSION_LIFETIME_DAYS": self.session_lifetime_days, "ENCRYPTION_ENABLED": self.encryption_enabled, "ENCRYPTION_MASTER_KEY_PATH": str(self.encryption_master_key_path), diff --git a/app/ui.py b/app/ui.py index 81e905e..695f5e2 100644 --- a/app/ui.py +++ b/app/ui.py @@ -1505,6 +1505,9 @@ def metrics_dashboard(): flash("Access denied: Metrics require admin permissions", "danger") return redirect(url_for("ui.buckets_overview")) + from app.version import APP_VERSION + import time + cpu_percent = psutil.cpu_percent(interval=0.1) memory = psutil.virtual_memory() @@ -1527,6 +1530,11 @@ def metrics_dashboard(): total_objects += stats.get("total_objects", stats.get("objects", 0)) total_bytes_used += stats.get("total_bytes", stats.get("bytes", 0)) total_versions += stats.get("version_count", 0) + + # Calculate system uptime + boot_time = psutil.boot_time() + uptime_seconds = time.time() - boot_time + uptime_days = int(uptime_seconds / 86400) return render_template( "metrics.html", @@ -1550,6 +1558,8 @@ def metrics_dashboard(): "versions": total_versions, "storage_used": _format_bytes(total_bytes_used), "storage_raw": total_bytes_used, + "version": APP_VERSION, + "uptime_days": uptime_days, } ) diff --git a/app/version.py b/app/version.py index 4a04f44..eb61b62 100644 --- a/app/version.py +++ b/app/version.py @@ -1,7 +1,7 @@ """Central location for the application version string.""" from __future__ import annotations -APP_VERSION = "0.1.3" +APP_VERSION = "0.1.4" def get_version() -> str: diff --git a/docs.md b/docs.md index eef7ef5..192d3cc 100644 --- a/docs.md +++ b/docs.md @@ -69,23 +69,97 @@ The repo now tracks a human-friendly release string inside `app/version.py` (see ## 3. Configuration Reference +All configuration is done via environment variables. The table below lists every supported variable. + +### Core Settings + | Variable | Default | Notes | | --- | --- | --- | | `STORAGE_ROOT` | `/data` | Filesystem home for all buckets/objects. | -| `MAX_UPLOAD_SIZE` | `1073741824` | Bytes. Caps incoming uploads in both API + UI. | +| `MAX_UPLOAD_SIZE` | `1073741824` (1 GiB) | Bytes. Caps incoming uploads in both API + UI. | | `UI_PAGE_SIZE` | `100` | `MaxKeys` hint shown in listings. | -| `SECRET_KEY` | `dev-secret-key` | Flask session key for UI auth. | -| `IAM_CONFIG` | `/data/.myfsio.sys/config/iam.json` | Stores users, secrets, and inline policies. | -| `BUCKET_POLICY_PATH` | `/data/.myfsio.sys/config/bucket_policies.json` | Bucket policy store (auto hot-reload). | -| `API_BASE_URL` | `None` | Used by the UI to hit API endpoints (presign/policy). If unset, the UI will auto-detect the host or use `X-Forwarded-*` headers. | +| `SECRET_KEY` | Auto-generated | Flask session key. Auto-generates and persists if not set. **Set explicitly in production.** | +| `API_BASE_URL` | `None` | Public URL for presigned URLs. Required behind proxies. | | `AWS_REGION` | `us-east-1` | Region embedded in SigV4 credential scope. | | `AWS_SERVICE` | `s3` | Service string for SigV4. | -| `ENCRYPTION_ENABLED` | `false` | Enable server-side encryption support. | -| `KMS_ENABLED` | `false` | Enable KMS key management for encryption. | -| `KMS_KEYS_PATH` | `data/kms_keys.json` | Path to store KMS key metadata. | -| `ENCRYPTION_MASTER_KEY_PATH` | `data/master.key` | Path to the master encryption key file. | -Set env vars (or pass overrides to `create_app`) to point the servers at custom paths. +### IAM & Security + +| Variable | Default | Notes | +| --- | --- | --- | +| `IAM_CONFIG` | `data/.myfsio.sys/config/iam.json` | Stores users, secrets, and inline policies. | +| `BUCKET_POLICY_PATH` | `data/.myfsio.sys/config/bucket_policies.json` | Bucket policy store (auto hot-reload). | +| `AUTH_MAX_ATTEMPTS` | `5` | Failed login attempts before lockout. | +| `AUTH_LOCKOUT_MINUTES` | `15` | Lockout duration after max failed attempts. | +| `SESSION_LIFETIME_DAYS` | `30` | How long UI sessions remain valid. | +| `SECRET_TTL_SECONDS` | `300` | TTL for ephemeral secrets (presigned URLs). | +| `UI_ENFORCE_BUCKET_POLICIES` | `false` | Whether the UI should enforce bucket policies. | + +### CORS (Cross-Origin Resource Sharing) + +| Variable | Default | Notes | +| --- | --- | --- | +| `CORS_ORIGINS` | `*` | Comma-separated allowed origins. Use specific domains in production. | +| `CORS_METHODS` | `GET,PUT,POST,DELETE,OPTIONS,HEAD` | Allowed HTTP methods. | +| `CORS_ALLOW_HEADERS` | `*` | Allowed request headers. | +| `CORS_EXPOSE_HEADERS` | `*` | Response headers visible to browsers (e.g., `ETag`). | + +### Rate Limiting + +| Variable | Default | Notes | +| --- | --- | --- | +| `RATE_LIMIT_DEFAULT` | `200 per minute` | Default rate limit for API endpoints. | +| `RATE_LIMIT_STORAGE_URI` | `memory://` | Storage backend for rate limits. Use `redis://host:port` for distributed setups. | + +### Logging + +| Variable | Default | Notes | +| --- | --- | --- | +| `LOG_LEVEL` | `INFO` | Log verbosity: `DEBUG`, `INFO`, `WARNING`, `ERROR`. | +| `LOG_TO_FILE` | `true` | Enable file logging. | +| `LOG_DIR` | `/logs` | Directory for log files. | +| `LOG_FILE` | `app.log` | Log filename. | +| `LOG_MAX_BYTES` | `5242880` (5 MB) | Max log file size before rotation. | +| `LOG_BACKUP_COUNT` | `3` | Number of rotated log files to keep. | + +### Encryption + +| Variable | Default | Notes | +| --- | --- | --- | +| `ENCRYPTION_ENABLED` | `false` | Enable server-side encryption support. | +| `ENCRYPTION_MASTER_KEY_PATH` | `data/.myfsio.sys/keys/master.key` | Path to the master encryption key file. | +| `DEFAULT_ENCRYPTION_ALGORITHM` | `AES256` | Default algorithm for new encrypted objects. | +| `KMS_ENABLED` | `false` | Enable KMS key management for encryption. | +| `KMS_KEYS_PATH` | `data/.myfsio.sys/keys/kms_keys.json` | Path to store KMS key metadata. | + +### Performance Tuning + +| Variable | Default | Notes | +| --- | --- | --- | +| `STREAM_CHUNK_SIZE` | `65536` (64 KB) | Chunk size for streaming large files. | +| `MULTIPART_MIN_PART_SIZE` | `5242880` (5 MB) | Minimum part size for multipart uploads. | +| `BUCKET_STATS_CACHE_TTL` | `60` | Seconds to cache bucket statistics. | +| `BULK_DELETE_MAX_KEYS` | `500` | Maximum keys per bulk delete request. | + +### Server Settings + +| Variable | Default | Notes | +| --- | --- | --- | +| `APP_HOST` | `0.0.0.0` | Network interface to bind to. | +| `APP_PORT` | `5000` | API server port (UI uses 5100). | +| `FLASK_DEBUG` | `0` | Enable Flask debug mode. **Never enable in production.** | + +### Production Checklist + +Before deploying to production, ensure you: + +1. **Set `SECRET_KEY`** - Use a strong, unique value (e.g., `openssl rand -base64 32`) +2. **Restrict CORS** - Set `CORS_ORIGINS` to your specific domains instead of `*` +3. **Configure `API_BASE_URL`** - Required for correct presigned URLs behind proxies +4. **Enable HTTPS** - Use a reverse proxy (nginx, Cloudflare) with TLS termination +5. **Review rate limits** - Adjust `RATE_LIMIT_DEFAULT` based on your needs +6. **Secure master keys** - Back up `ENCRYPTION_MASTER_KEY_PATH` if using encryption +7. **Use `--prod` flag** - Runs with Waitress instead of Flask dev server ### Proxy Configuration @@ -95,6 +169,333 @@ If running behind a reverse proxy (e.g., Nginx, Cloudflare, or a tunnel), ensure The application automatically trusts these headers to generate correct presigned URLs (e.g., `https://s3.example.com/...` instead of `http://127.0.0.1:5000/...`). Alternatively, you can explicitly set `API_BASE_URL` to your public endpoint. +## 4. Upgrading and Updates + +### Version Checking + +The application version is tracked in `app/version.py` and exposed via: +- **Health endpoint:** `GET /healthz` returns JSON with `version` field +- **Metrics dashboard:** Navigate to `/ui/metrics` to see the running version in the System Status card + +To check your current version: + +```bash +# API health endpoint +curl http://localhost:5000/healthz + +# Or inspect version.py directly +cat app/version.py | grep APP_VERSION +``` + +### Pre-Update Backup Procedures + +**Always backup before upgrading to prevent data loss:** + +```bash +# 1. Stop the application +# Ctrl+C if running in terminal, or: +docker stop myfsio # if using Docker + +# 2. Backup configuration files (CRITICAL) +mkdir -p backups/$(date +%Y%m%d_%H%M%S) +cp -r data/.myfsio.sys/config backups/$(date +%Y%m%d_%H%M%S)/ + +# 3. Backup all data (optional but recommended) +tar -czf backups/data_$(date +%Y%m%d_%H%M%S).tar.gz data/ + +# 4. Backup logs for audit trail +cp -r logs backups/$(date +%Y%m%d_%H%M%S)/ +``` + +**Windows PowerShell:** + +```powershell +# Create timestamped backup +$timestamp = Get-Date -Format "yyyyMMdd_HHmmss" +New-Item -ItemType Directory -Path "backups\$timestamp" -Force + +# Backup configs +Copy-Item -Recurse "data\.myfsio.sys\config" "backups\$timestamp\" + +# Backup entire data directory +Compress-Archive -Path "data\" -DestinationPath "backups\data_$timestamp.zip" +``` + +**Critical files to backup:** +- `data/.myfsio.sys/config/iam.json` – User accounts and access keys +- `data/.myfsio.sys/config/bucket_policies.json` – Bucket access policies +- `data/.myfsio.sys/config/kms_keys.json` – Encryption keys (if using KMS) +- `data/.myfsio.sys/config/secret_store.json` – Application secrets + +### Update Procedures + +#### Source Installation Updates + +```bash +# 1. Backup (see above) +# 2. Pull latest code +git fetch origin +git checkout main # or your target branch/tag +git pull + +# 3. Check for dependency changes +pip install -r requirements.txt + +# 4. Review CHANGELOG/release notes for breaking changes +cat CHANGELOG.md # if available + +# 5. Run migration scripts (if any) +# python scripts/migrate_vX_to_vY.py # example + +# 6. Restart application +python run.py +``` + +#### Docker Updates + +```bash +# 1. Backup (see above) +# 2. Pull/rebuild image +docker pull yourregistry/myfsio:latest +# OR rebuild from source: +docker build -t myfsio:latest . + +# 3. Stop and remove old container +docker stop myfsio +docker rm myfsio + +# 4. Start new container with same volumes +docker run -d \ + --name myfsio \ + -p 5000:5000 -p 5100:5100 \ + -v "$(pwd)/data:/app/data" \ + -v "$(pwd)/logs:/app/logs" \ + -e SECRET_KEY="your-secret" \ + myfsio:latest + +# 5. Verify health +curl http://localhost:5000/healthz +``` + +### Version Compatibility Checks + +Before upgrading across major versions, verify compatibility: + +| From Version | To Version | Breaking Changes | Migration Required | +|--------------|------------|------------------|-------------------| +| 0.1.x | 0.2.x | None expected | No | +| < 0.1.0 | >= 0.1.0 | New IAM config format | Yes - run migration script | + +**Automatic compatibility detection:** + +The application will log warnings on startup if config files need migration: + +``` +WARNING: IAM config format is outdated (v1). Please run: python scripts/migrate_iam.py +``` + +**Manual compatibility check:** + +```bash +# Compare version schemas +python -c "from app.version import APP_VERSION; print(f'Running: {APP_VERSION}')" +python scripts/check_compatibility.py data/.myfsio.sys/config/ +``` + +### Migration Steps for Breaking Changes + +When release notes indicate breaking changes, follow these steps: + +#### Config Format Migrations + +```bash +# 1. Backup first (critical!) +cp data/.myfsio.sys/config/iam.json data/.myfsio.sys/config/iam.json.backup + +# 2. Run provided migration script +python scripts/migrate_iam_v1_to_v2.py + +# 3. Validate migration +python scripts/validate_config.py + +# 4. Test with read-only mode first (if available) +# python run.py --read-only + +# 5. Restart normally +python run.py +``` + +#### Database/Storage Schema Changes + +If object metadata format changes: + +```bash +# 1. Run storage migration script +python scripts/migrate_storage.py --dry-run # preview changes + +# 2. Apply migration +python scripts/migrate_storage.py --apply + +# 3. Verify integrity +python scripts/verify_storage.py +``` + +#### IAM Policy Updates + +If IAM action names change (e.g., `s3:Get` → `s3:GetObject`): + +```bash +# Migration script will update all policies +python scripts/migrate_policies.py \ + --input data/.myfsio.sys/config/iam.json \ + --backup data/.myfsio.sys/config/iam.json.v1 + +# Review changes before committing +python scripts/diff_policies.py \ + data/.myfsio.sys/config/iam.json.v1 \ + data/.myfsio.sys/config/iam.json +``` + +### Rollback Procedures + +If an update causes issues, rollback to the previous version: + +#### Quick Rollback (Source) + +```bash +# 1. Stop application +# Ctrl+C or kill process + +# 2. Revert code +git checkout +# OR +git reset --hard HEAD~1 + +# 3. Restore configs from backup +cp backups/20241213_103000/config/* data/.myfsio.sys/config/ + +# 4. Downgrade dependencies if needed +pip install -r requirements.txt + +# 5. Restart +python run.py +``` + +#### Docker Rollback + +```bash +# 1. Stop current container +docker stop myfsio +docker rm myfsio + +# 2. Start previous version +docker run -d \ + --name myfsio \ + -p 5000:5000 -p 5100:5100 \ + -v "$(pwd)/data:/app/data" \ + -v "$(pwd)/logs:/app/logs" \ + -e SECRET_KEY="your-secret" \ + myfsio:0.1.3 # specify previous version tag + +# 3. Verify +curl http://localhost:5000/healthz +``` + +#### Emergency Config Restore + +If only config is corrupted but code is fine: + +```bash +# Stop app +# Restore from latest backup +cp backups/20241213_103000/config/iam.json data/.myfsio.sys/config/ +cp backups/20241213_103000/config/bucket_policies.json data/.myfsio.sys/config/ + +# Restart app +python run.py +``` + +### Blue-Green Deployment (Zero Downtime) + +For production environments requiring zero downtime: + +```bash +# 1. Run new version on different port (e.g., 5001/5101) +APP_PORT=5001 UI_PORT=5101 python run.py & + +# 2. Health check new instance +curl http://localhost:5001/healthz + +# 3. Update load balancer to route to new ports + +# 4. Monitor for issues + +# 5. Gracefully stop old instance +kill -SIGTERM +``` + +### Post-Update Verification + +After any update, verify functionality: + +```bash +# 1. Health check +curl http://localhost:5000/healthz + +# 2. Login to UI +open http://localhost:5100/ui + +# 3. Test IAM authentication +curl -H "X-Amz-Security-Token: :" \ + http://localhost:5000/ + +# 4. Test presigned URL generation +# Via UI or API + +# 5. Check logs for errors +tail -n 100 logs/myfsio.log +``` + +### Automated Update Scripts + +Create a custom update script for your environment: + +```bash +#!/bin/bash +# update.sh - Automated update with rollback capability + +set -e # Exit on error + +VERSION_NEW="$1" +BACKUP_DIR="backups/$(date +%Y%m%d_%H%M%S)" + +echo "Creating backup..." +mkdir -p "$BACKUP_DIR" +cp -r data/.myfsio.sys/config "$BACKUP_DIR/" + +echo "Updating to version $VERSION_NEW..." +git fetch origin +git checkout "v$VERSION_NEW" +pip install -r requirements.txt + +echo "Starting application..." +python run.py & +APP_PID=$! + +# Wait and health check +sleep 5 +if curl -f http://localhost:5000/healthz; then + echo "Update successful!" +else + echo "Health check failed, rolling back..." + kill $APP_PID + git checkout - + cp -r "$BACKUP_DIR/config/*" data/.myfsio.sys/config/ + python run.py & + exit 1 +fi +``` + ## 4. Authentication & IAM 1. On first boot, `data/.myfsio.sys/config/iam.json` is seeded with `localadmin / localadmin` that has wildcard access. @@ -577,9 +978,3 @@ DELETE /bucket-policy/ # Delete policy GET /?quota # Get bucket quota PUT /?quota # Set bucket quota (admin only) ``` - -## 14. Next Steps - -- Tailor IAM + policy JSON files for team-ready presets. -- Wrap `run_api.py` with gunicorn or another WSGI server for long-running workloads. -- Extend `bucket_policies.json` to cover Deny statements that simulate production security controls. diff --git a/templates/docs.html b/templates/docs.html index fd7935c..2b2e325 100644 --- a/templates/docs.html +++ b/templates/docs.html @@ -55,8 +55,8 @@ python run.py --mode ui API_BASE_URL - http://127.0.0.1:5000 - The public URL of the API. Required if running behind a proxy or if the UI and API are on different domains. Ensures presigned URLs are generated correctly. + None + The public URL of the API. Required if running behind a proxy. Ensures presigned URLs are generated correctly. STORAGE_ROOT @@ -65,13 +65,13 @@ python run.py --mode ui MAX_UPLOAD_SIZE - 5 GB - Max request body size. + 1 GB + Max request body size in bytes. SECRET_KEY - (Random) - Flask session key. Set this in production. + (Auto-generated) + Flask session key. Auto-generates if not set. Set explicitly in production. APP_HOST @@ -81,7 +81,51 @@ python run.py --mode ui APP_PORT 5000 - Listen port. + Listen port (UI uses 5100). + + + CORS Settings + + + CORS_ORIGINS + * + Allowed origins. Restrict in production. + + + CORS_METHODS + GET,PUT,POST,DELETE,OPTIONS,HEAD + Allowed HTTP methods. + + + CORS_ALLOW_HEADERS + * + Allowed request headers. + + + CORS_EXPOSE_HEADERS + * + Response headers visible to browsers (e.g., ETag). + + + Security Settings + + + AUTH_MAX_ATTEMPTS + 5 + Failed login attempts before lockout. + + + AUTH_LOCKOUT_MINUTES + 15 + Lockout duration after max failed attempts. + + + RATE_LIMIT_DEFAULT + 200 per minute + Default API rate limit. + + + Encryption Settings ENCRYPTION_ENABLED @@ -93,9 +137,25 @@ python run.py --mode ui false Enable KMS key management for encryption. + + Logging Settings + + + LOG_LEVEL + INFO + Log verbosity: DEBUG, INFO, WARNING, ERROR. + + + LOG_TO_FILE + true + Enable file logging. + +
+ Production Checklist: Set SECRET_KEY, restrict CORS_ORIGINS, configure API_BASE_URL, enable HTTPS via reverse proxy, and use --prod flag. +
diff --git a/templates/metrics.html b/templates/metrics.html index 2fb7543..b786ee3 100644 --- a/templates/metrics.html +++ b/templates/metrics.html @@ -126,7 +126,6 @@
System Overview
- Live
@@ -233,14 +232,14 @@ - Healthy + v{{ app.version }}

System Status

All systems operational. Your storage infrastructure is running smoothly with no detected issues.

-
99.9%
+
{{ app.uptime_days }}d
Uptime