2906 lines
137 KiB
HTML
2906 lines
137 KiB
HTML
{% extends "base.html" %}
|
||
{% block content %}
|
||
<section class="docs-hero p-4 p-md-5 rounded-4 mb-4">
|
||
<div class="d-flex flex-column flex-lg-row justify-content-between gap-3">
|
||
<div>
|
||
<p class="text-uppercase fw-semibold small text-white-50 mb-2">Documentation</p>
|
||
<h1 class="display-6 fw-semibold mb-2">Your guide to MyFSIO</h1>
|
||
<p class="lead mb-0 text-light">Follow these steps to install, authenticate, master the console, and automate everything through the API.</p>
|
||
</div>
|
||
<div class="docs-callout text-light">
|
||
<div class="small text-uppercase opacity-75">API base URL</div>
|
||
<code class="fs-6 text-wrap">{{ api_base }}</code>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
<div class="row g-4">
|
||
<div class="col-12 d-xl-none">
|
||
<div class="card shadow-sm docs-sidebar-mobile mb-0">
|
||
<div class="card-body py-3">
|
||
<div class="d-flex align-items-center justify-content-between mb-2">
|
||
<h3 class="h6 text-uppercase text-muted mb-0">On this page</h3>
|
||
<button class="btn btn-sm btn-outline-secondary" type="button" data-bs-toggle="collapse" data-bs-target="#mobileDocsToc" aria-expanded="false" aria-controls="mobileDocsToc">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" viewBox="0 0 16 16">
|
||
<path fill-rule="evenodd" d="M1.646 4.646a.5.5 0 0 1 .708 0L8 10.293l5.646-5.647a.5.5 0 0 1 .708.708l-6 6a.5.5 0 0 1-.708 0l-6-6a.5.5 0 0 1 0-.708z"/>
|
||
</svg>
|
||
</button>
|
||
</div>
|
||
<div class="collapse" id="mobileDocsToc">
|
||
<ul class="list-unstyled docs-toc mb-0 small">
|
||
<li><a href="#setup">Set up & run</a></li>
|
||
<li><a href="#background">Running in background</a></li>
|
||
<li><a href="#auth">Authentication & IAM</a></li>
|
||
<li><a href="#console">Console tour</a></li>
|
||
<li><a href="#automation">Automation / CLI</a></li>
|
||
<li><a href="#api">REST endpoints</a></li>
|
||
<li><a href="#examples">API Examples</a></li>
|
||
<li><a href="#replication">Site Replication & Sync</a></li>
|
||
<li><a href="#site-registry">Site Registry</a></li>
|
||
<li><a href="#versioning">Object Versioning</a></li>
|
||
<li><a href="#quotas">Bucket Quotas</a></li>
|
||
<li><a href="#encryption">Encryption</a></li>
|
||
<li><a href="#lifecycle">Lifecycle Rules</a></li>
|
||
<li><a href="#garbage-collection">Garbage Collection</a></li>
|
||
<li><a href="#integrity">Integrity Scanner</a></li>
|
||
<li><a href="#metrics">Metrics History</a></li>
|
||
<li><a href="#operation-metrics">Operation Metrics</a></li>
|
||
<li><a href="#troubleshooting">Troubleshooting</a></li>
|
||
<li><a href="#health-check">Health Check</a></li>
|
||
<li><a href="#object-lock">Object Lock & Retention</a></li>
|
||
<li><a href="#access-logging">Access Logging</a></li>
|
||
<li><a href="#notifications">Notifications & Webhooks</a></li>
|
||
<li><a href="#select-content">SelectObjectContent</a></li>
|
||
<li><a href="#advanced-ops">Advanced Operations</a></li>
|
||
<li><a href="#acls">Access Control Lists</a></li>
|
||
<li><a href="#tagging">Object & Bucket Tagging</a></li>
|
||
<li><a href="#website-hosting">Static Website Hosting</a></li>
|
||
<li><a href="#cors-config">CORS Configuration</a></li>
|
||
<li><a href="#post-object">PostObject (Form Upload)</a></li>
|
||
<li><a href="#list-objects-v2">List Objects API v2</a></li>
|
||
<li><a href="#upgrading">Upgrading & Updates</a></li>
|
||
<li><a href="#api-matrix">Full API Reference</a></li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="col-xl-8">
|
||
<article id="setup" class="card shadow-sm docs-section">
|
||
<div class="card-body">
|
||
<div class="d-flex align-items-center gap-2 mb-3">
|
||
<span class="docs-section-kicker">01</span>
|
||
<h2 class="h4 mb-0">Set up & run locally</h2>
|
||
</div>
|
||
<p class="text-muted">Prepare a virtual environment, install dependencies, and launch both servers for a complete console + API experience.</p>
|
||
<ol class="docs-steps">
|
||
<li>Install Python 3.11+ plus system build tools.</li>
|
||
<li>Create a virtual environment and install <code>requirements.txt</code>.</li>
|
||
<li>Start the services with <code>python run.py</code>.</li>
|
||
</ol>
|
||
<pre class="mb-3"><code class="language-bash">python -m venv .venv
|
||
. .venv/Scripts/activate # PowerShell: .\\.venv\\Scripts\\Activate.ps1
|
||
pip install -r requirements.txt
|
||
|
||
# Run both API and UI (Development)
|
||
python run.py
|
||
|
||
# Run in Production (Granian server)
|
||
python run.py --prod
|
||
|
||
# Or run individually
|
||
python run.py --mode api
|
||
python run.py --mode ui
|
||
</code></pre>
|
||
<h3 class="h6 mt-4 mb-2">Configuration</h3>
|
||
<p class="text-muted small">Configuration defaults live in <code>app/config.py</code>. You can override them using environment variables. This is critical for production deployments behind proxies.</p>
|
||
<div class="table-responsive">
|
||
<table class="table table-sm table-bordered small mb-0">
|
||
<thead class="table-light">
|
||
<tr>
|
||
<th style="min-width: 180px;">Variable</th>
|
||
<th style="min-width: 120px;">Default</th>
|
||
<th class="text-wrap" style="min-width: 250px;">Description</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr>
|
||
<td><code>API_BASE_URL</code></td>
|
||
<td><code>http://127.0.0.1:5000</code></td>
|
||
<td>Internal S3 API URL used by the web UI proxy. Also used for presigned URL generation. Set to your public URL if running behind a reverse proxy.</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>STORAGE_ROOT</code></td>
|
||
<td><code>./data</code></td>
|
||
<td>Directory for buckets and objects.</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>MAX_UPLOAD_SIZE</code></td>
|
||
<td><code>1 GB</code></td>
|
||
<td>Max request body size in bytes.</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>SECRET_KEY</code></td>
|
||
<td>(Auto-generated)</td>
|
||
<td>Flask session key. Auto-generates if not set. <strong>Set explicitly in production.</strong></td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>APP_HOST</code></td>
|
||
<td><code>0.0.0.0</code></td>
|
||
<td>Bind interface.</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>APP_PORT</code></td>
|
||
<td><code>5000</code></td>
|
||
<td>Listen port (UI uses 5100).</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>DISPLAY_TIMEZONE</code></td>
|
||
<td><code>UTC</code></td>
|
||
<td>Timezone for UI timestamps (e.g., <code>US/Eastern</code>, <code>Asia/Tokyo</code>).</td>
|
||
</tr>
|
||
<tr class="table-secondary">
|
||
<td colspan="3" class="fw-semibold">CORS Settings</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>CORS_ORIGINS</code></td>
|
||
<td><code>*</code></td>
|
||
<td>Allowed origins. <strong>Restrict in production.</strong></td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>CORS_METHODS</code></td>
|
||
<td><code>GET,PUT,POST,DELETE,OPTIONS,HEAD</code></td>
|
||
<td>Allowed HTTP methods.</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>CORS_ALLOW_HEADERS</code></td>
|
||
<td><code>*</code></td>
|
||
<td>Allowed request headers.</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>CORS_EXPOSE_HEADERS</code></td>
|
||
<td><code>*</code></td>
|
||
<td>Response headers visible to browsers (e.g., <code>ETag</code>).</td>
|
||
</tr>
|
||
<tr class="table-secondary">
|
||
<td colspan="3" class="fw-semibold">Security Settings</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>AUTH_MAX_ATTEMPTS</code></td>
|
||
<td><code>5</code></td>
|
||
<td>Failed login attempts before lockout.</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>AUTH_LOCKOUT_MINUTES</code></td>
|
||
<td><code>15</code></td>
|
||
<td>Lockout duration after max failed attempts.</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>RATE_LIMIT_DEFAULT</code></td>
|
||
<td><code>200 per minute</code></td>
|
||
<td>Default API rate limit.</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>RATE_LIMIT_LIST_BUCKETS</code></td>
|
||
<td><code>60 per minute</code></td>
|
||
<td>Rate limit for listing buckets.</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>RATE_LIMIT_BUCKET_OPS</code></td>
|
||
<td><code>120 per minute</code></td>
|
||
<td>Rate limit for bucket operations.</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>RATE_LIMIT_OBJECT_OPS</code></td>
|
||
<td><code>240 per minute</code></td>
|
||
<td>Rate limit for object operations.</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>RATE_LIMIT_HEAD_OPS</code></td>
|
||
<td><code>100 per minute</code></td>
|
||
<td>Rate limit for HEAD requests.</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>RATE_LIMIT_ADMIN</code></td>
|
||
<td><code>60 per minute</code></td>
|
||
<td>Rate limit for admin API endpoints (<code>/admin/*</code>).</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>ADMIN_ACCESS_KEY</code></td>
|
||
<td>(none)</td>
|
||
<td>Custom access key for the admin user on first run or credential reset. Random if unset.</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>ADMIN_SECRET_KEY</code></td>
|
||
<td>(none)</td>
|
||
<td>Custom secret key for the admin user on first run or credential reset. Random if unset.</td>
|
||
</tr>
|
||
<tr class="table-secondary">
|
||
<td colspan="3" class="fw-semibold">Server Settings</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>SERVER_THREADS</code></td>
|
||
<td><code>0</code> (auto)</td>
|
||
<td>Granian blocking threads (1-64). 0 = auto (CPU cores × 2).</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>SERVER_CONNECTION_LIMIT</code></td>
|
||
<td><code>0</code> (auto)</td>
|
||
<td>Max concurrent connections (10-1000). 0 = auto (RAM-based).</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>SERVER_BACKLOG</code></td>
|
||
<td><code>0</code> (auto)</td>
|
||
<td>TCP listen backlog (64-4096). 0 = auto (conn_limit × 2).</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>SERVER_CHANNEL_TIMEOUT</code></td>
|
||
<td><code>120</code></td>
|
||
<td>Idle connection timeout in seconds (10-300).</td>
|
||
</tr>
|
||
<tr class="table-secondary">
|
||
<td colspan="3" class="fw-semibold">Encryption Settings</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>ENCRYPTION_ENABLED</code></td>
|
||
<td><code>false</code></td>
|
||
<td>Enable server-side encryption support.</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>KMS_ENABLED</code></td>
|
||
<td><code>false</code></td>
|
||
<td>Enable KMS key management for encryption.</td>
|
||
</tr>
|
||
<tr class="table-secondary">
|
||
<td colspan="3" class="fw-semibold">Logging Settings</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>LOG_LEVEL</code></td>
|
||
<td><code>INFO</code></td>
|
||
<td>Log verbosity: DEBUG, INFO, WARNING, ERROR.</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>LOG_TO_FILE</code></td>
|
||
<td><code>true</code></td>
|
||
<td>Enable file logging.</td>
|
||
</tr>
|
||
<tr class="table-secondary">
|
||
<td colspan="3" class="fw-semibold">Metrics History Settings</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>METRICS_HISTORY_ENABLED</code></td>
|
||
<td><code>false</code></td>
|
||
<td>Enable metrics history recording and charts (opt-in).</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>METRICS_HISTORY_RETENTION_HOURS</code></td>
|
||
<td><code>24</code></td>
|
||
<td>How long to retain metrics history data.</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>METRICS_HISTORY_INTERVAL_MINUTES</code></td>
|
||
<td><code>5</code></td>
|
||
<td>Interval between history snapshots.</td>
|
||
</tr>
|
||
<tr class="table-secondary">
|
||
<td colspan="3" class="fw-semibold">Site Sync Settings (Bidirectional Replication)</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>SITE_SYNC_ENABLED</code></td>
|
||
<td><code>false</code></td>
|
||
<td>Enable bi-directional site sync background worker.</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>SITE_SYNC_INTERVAL_SECONDS</code></td>
|
||
<td><code>60</code></td>
|
||
<td>Interval between sync cycles (seconds).</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>SITE_SYNC_BATCH_SIZE</code></td>
|
||
<td><code>100</code></td>
|
||
<td>Max objects to pull per sync cycle.</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>SITE_SYNC_CONNECT_TIMEOUT_SECONDS</code></td>
|
||
<td><code>10</code></td>
|
||
<td>Connection timeout for site sync (seconds).</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>SITE_SYNC_READ_TIMEOUT_SECONDS</code></td>
|
||
<td><code>120</code></td>
|
||
<td>Read timeout for site sync (seconds).</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>SITE_SYNC_MAX_RETRIES</code></td>
|
||
<td><code>2</code></td>
|
||
<td>Max retry attempts for site sync operations.</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>SITE_SYNC_CLOCK_SKEW_TOLERANCE_SECONDS</code></td>
|
||
<td><code>1.0</code></td>
|
||
<td>Clock skew tolerance for conflict resolution.</td>
|
||
</tr>
|
||
<tr class="table-secondary">
|
||
<td colspan="3" class="fw-semibold">Replication Settings</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>REPLICATION_CONNECT_TIMEOUT_SECONDS</code></td>
|
||
<td><code>5</code></td>
|
||
<td>Connection timeout for replication (seconds).</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>REPLICATION_READ_TIMEOUT_SECONDS</code></td>
|
||
<td><code>30</code></td>
|
||
<td>Read timeout for replication (seconds).</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>REPLICATION_MAX_RETRIES</code></td>
|
||
<td><code>2</code></td>
|
||
<td>Max retry attempts for replication operations.</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>REPLICATION_STREAMING_THRESHOLD_BYTES</code></td>
|
||
<td><code>10485760</code></td>
|
||
<td>Objects larger than this use streaming upload (10 MB).</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>REPLICATION_MAX_FAILURES_PER_BUCKET</code></td>
|
||
<td><code>50</code></td>
|
||
<td>Max failure records to keep per bucket.</td>
|
||
</tr>
|
||
<tr class="table-secondary">
|
||
<td colspan="3" class="fw-semibold">Security & Auth Settings</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>SIGV4_TIMESTAMP_TOLERANCE_SECONDS</code></td>
|
||
<td><code>900</code></td>
|
||
<td>Max time skew for SigV4 requests (15 minutes).</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>PRESIGNED_URL_MIN_EXPIRY_SECONDS</code></td>
|
||
<td><code>1</code></td>
|
||
<td>Minimum presigned URL expiry time.</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>PRESIGNED_URL_MAX_EXPIRY_SECONDS</code></td>
|
||
<td><code>604800</code></td>
|
||
<td>Maximum presigned URL expiry time (7 days).</td>
|
||
</tr>
|
||
<tr class="table-secondary">
|
||
<td colspan="3" class="fw-semibold">Proxy & Network Settings</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>NUM_TRUSTED_PROXIES</code></td>
|
||
<td><code>1</code></td>
|
||
<td>Number of trusted reverse proxies for <code>X-Forwarded-*</code> headers.</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>ALLOWED_REDIRECT_HOSTS</code></td>
|
||
<td>(empty)</td>
|
||
<td>Comma-separated whitelist of safe redirect targets.</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>ALLOW_INTERNAL_ENDPOINTS</code></td>
|
||
<td><code>false</code></td>
|
||
<td>Allow connections to internal/private IPs (webhooks, replication).</td>
|
||
</tr>
|
||
<tr class="table-secondary">
|
||
<td colspan="3" class="fw-semibold">Storage Limits</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>OBJECT_KEY_MAX_LENGTH_BYTES</code></td>
|
||
<td><code>1024</code></td>
|
||
<td>Maximum object key length in bytes.</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>OBJECT_CACHE_MAX_SIZE</code></td>
|
||
<td><code>100</code></td>
|
||
<td>Maximum number of objects in cache.</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>BUCKET_CONFIG_CACHE_TTL_SECONDS</code></td>
|
||
<td><code>30</code></td>
|
||
<td>Bucket config cache TTL in seconds.</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>OBJECT_TAG_LIMIT</code></td>
|
||
<td><code>50</code></td>
|
||
<td>Maximum number of tags per object.</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>LIFECYCLE_MAX_HISTORY_PER_BUCKET</code></td>
|
||
<td><code>50</code></td>
|
||
<td>Max lifecycle history records per bucket.</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>OBJECT_CACHE_TTL</code></td>
|
||
<td><code>60</code></td>
|
||
<td>Seconds to cache object metadata.</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>BULK_DOWNLOAD_MAX_BYTES</code></td>
|
||
<td><code>1 GB</code></td>
|
||
<td>Max total size for bulk ZIP downloads.</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>ENCRYPTION_CHUNK_SIZE_BYTES</code></td>
|
||
<td><code>65536</code></td>
|
||
<td>Chunk size for streaming encryption (64 KB).</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>KMS_GENERATE_DATA_KEY_MIN_BYTES</code></td>
|
||
<td><code>1</code></td>
|
||
<td>Minimum data key size for KMS generation.</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>KMS_GENERATE_DATA_KEY_MAX_BYTES</code></td>
|
||
<td><code>1024</code></td>
|
||
<td>Maximum data key size for KMS generation.</td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
</div>
|
||
<div class="alert alert-warning mt-3 mb-0 small">
|
||
<strong>Production Checklist:</strong> Set <code>SECRET_KEY</code> (also enables IAM config encryption at rest), restrict <code>CORS_ORIGINS</code>, configure <code>API_BASE_URL</code>, enable HTTPS via reverse proxy, use <code>--prod</code> flag, and set credential expiry on non-admin users.
|
||
</div>
|
||
</div>
|
||
</article>
|
||
<article id="background" class="card shadow-sm docs-section">
|
||
<div class="card-body">
|
||
<div class="d-flex align-items-center gap-2 mb-3">
|
||
<span class="docs-section-kicker">02</span>
|
||
<h2 class="h4 mb-0">Running in background</h2>
|
||
</div>
|
||
<p class="text-muted">For production or server deployments, run MyFSIO as a background service so it persists after you close the terminal.</p>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Quick Start (nohup)</h3>
|
||
<p class="text-muted small">Simplest way to run in background—survives terminal close:</p>
|
||
<pre class="mb-3"><code class="language-bash"># Using Python
|
||
nohup python run.py --prod > /dev/null 2>&1 &
|
||
|
||
# Using compiled binary
|
||
nohup ./myfsio > /dev/null 2>&1 &
|
||
|
||
# Check if running
|
||
ps aux | grep myfsio</code></pre>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Screen / Tmux</h3>
|
||
<p class="text-muted small">Attach/detach from a persistent session:</p>
|
||
<pre class="mb-3"><code class="language-bash"># Start in a detached screen session
|
||
screen -dmS myfsio ./myfsio
|
||
|
||
# Attach to view logs
|
||
screen -r myfsio
|
||
|
||
# Detach: press Ctrl+A, then D</code></pre>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Systemd (Recommended for Production)</h3>
|
||
<p class="text-muted small">Create <code>/etc/systemd/system/myfsio.service</code>:</p>
|
||
<pre class="mb-3"><code class="language-ini">[Unit]
|
||
Description=MyFSIO S3-Compatible Storage
|
||
After=network.target
|
||
|
||
[Service]
|
||
Type=simple
|
||
User=myfsio
|
||
WorkingDirectory=/opt/myfsio
|
||
ExecStart=/opt/myfsio/myfsio
|
||
Restart=on-failure
|
||
RestartSec=5
|
||
Environment=STORAGE_ROOT=/var/lib/myfsio
|
||
Environment=API_BASE_URL=https://s3.example.com
|
||
|
||
[Install]
|
||
WantedBy=multi-user.target</code></pre>
|
||
<p class="text-muted small">Then enable and start:</p>
|
||
<pre class="mb-0"><code class="language-bash">sudo systemctl daemon-reload
|
||
sudo systemctl enable myfsio
|
||
sudo systemctl start myfsio
|
||
|
||
# Check status
|
||
sudo systemctl status myfsio
|
||
sudo journalctl -u myfsio -f # View logs</code></pre>
|
||
</div>
|
||
</article>
|
||
<article id="auth" class="card shadow-sm docs-section">
|
||
<div class="card-body">
|
||
<div class="d-flex align-items-center gap-2 mb-3">
|
||
<span class="docs-section-kicker">03</span>
|
||
<h2 class="h4 mb-0">Authenticate & manage IAM</h2>
|
||
</div>
|
||
<p class="text-muted">On first startup, MyFSIO generates random admin credentials and prints them to the console. Set <code>ADMIN_ACCESS_KEY</code> and <code>ADMIN_SECRET_KEY</code> env vars for custom credentials. When <code>SECRET_KEY</code> is configured, the IAM config is encrypted at rest. To reset credentials, run <code>python run.py --reset-cred</code>.</p>
|
||
<div class="docs-highlight mb-3">
|
||
<ol class="mb-0">
|
||
<li>Check the console output for the generated <code>Access Key</code> and <code>Secret Key</code>, then visit <code>/ui/login</code>.</li>
|
||
<li>Create additional users with descriptive display names, AWS-style inline policies (for example <code>{"bucket": "*", "actions": ["list", "read"]}</code>), and optional credential expiry dates.</li>
|
||
<li>Set credential expiry on users to grant time-limited access. The UI shows expiry badges and provides preset durations (1h, 24h, 7d, 30d, 90d). Expired credentials are rejected at authentication.</li>
|
||
<li>Rotate secrets when sharing with CI jobs—new secrets display once and persist to <code>data/.myfsio.sys/config/iam.json</code>.</li>
|
||
<li>Bucket policies layer on top of IAM. Apply Private/Public presets or paste custom JSON; changes reload instantly.</li>
|
||
</ol>
|
||
</div>
|
||
<p class="mb-0 text-muted">All API calls require <code>X-Access-Key</code> and <code>X-Secret-Key</code> headers. The UI stores them in the Flask session after you log in.</p>
|
||
</div>
|
||
</article>
|
||
<article id="console" class="card shadow-sm docs-section">
|
||
<div class="card-body">
|
||
<div class="d-flex align-items-center gap-2 mb-3">
|
||
<span class="docs-section-kicker">04</span>
|
||
<h2 class="h4 mb-0">Use the console effectively</h2>
|
||
</div>
|
||
<p class="text-muted">Each workspace models an S3 workflow so you can administer buckets end-to-end.</p>
|
||
<div class="docs-pill-list">
|
||
<div>
|
||
<h3 class="h6 text-uppercase text-muted">Buckets</h3>
|
||
<ul>
|
||
<li>Create/delete buckets from the overview. Badges reveal IAM-only, public-read, or custom-policy states.</li>
|
||
<li>Summary stats show live object counts and total capacity; click through for inventories.</li>
|
||
</ul>
|
||
</div>
|
||
<div>
|
||
<h3 class="h6 text-uppercase text-muted">Uploads</h3>
|
||
<ul>
|
||
<li>Drag and drop folders or files into the upload modal. Objects above 16 MB switch to multipart automatically.</li>
|
||
<li>Progress rows highlight retries, throughput, and completion even if you close the modal.</li>
|
||
</ul>
|
||
</div>
|
||
<div>
|
||
<h3 class="h6 text-uppercase text-muted">Object browser</h3>
|
||
<ul>
|
||
<li>Navigate folder hierarchies using breadcrumbs. Objects with <code>/</code> in keys display as folders.</li>
|
||
<li>Infinite scroll loads more objects automatically. Choose batch size (50–250) from the footer dropdown.</li>
|
||
<li>Bulk select objects for multi-delete or multi-download (ZIP archive, up to 1 GiB). Filter by name using the search box.</li>
|
||
<li>If loading fails, click <strong>Retry</strong> to attempt again—no page refresh needed.</li>
|
||
</ul>
|
||
</div>
|
||
<div>
|
||
<h3 class="h6 text-uppercase text-muted">Object details</h3>
|
||
<ul>
|
||
<li>Selecting an object opens the preview card with metadata, inline viewers, presign generator, and version history.</li>
|
||
<li>Trigger downloads, deletes, restores, or metadata refreshes without leaving the panel.</li>
|
||
</ul>
|
||
</div>
|
||
<div>
|
||
<h3 class="h6 text-uppercase text-muted">Policies & versioning</h3>
|
||
<ul>
|
||
<li>Toggle versioning (requires write access). Archived-only keys are flagged so you can restore them quickly.</li>
|
||
<li>The policy editor saves drafts, ships with presets, and hot-reloads <code>data/.myfsio.sys/config/bucket_policies.json</code>.</li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</article>
|
||
<article id="automation" class="card shadow-sm docs-section">
|
||
<div class="card-body">
|
||
<div class="d-flex align-items-center gap-2 mb-3">
|
||
<span class="docs-section-kicker">05</span>
|
||
<h2 class="h4 mb-0">Automate with CLI & tools</h2>
|
||
</div>
|
||
<p class="text-muted">Point standard S3 clients at {{ api_base }} and reuse the same IAM credentials.</p>
|
||
<div class="mb-4">
|
||
<h3 class="h6 text-uppercase text-muted">AWS CLI</h3>
|
||
<pre class="mb-3"><code class="language-bash">aws configure set aws_access_key_id <access_key>
|
||
aws configure set aws_secret_access_key <secret_key>
|
||
aws configure set default.region us-east-1
|
||
|
||
aws --endpoint-url {{ api_base }} s3 ls
|
||
aws --endpoint-url {{ api_base }} s3api create-bucket --bucket demo
|
||
aws --endpoint-url {{ api_base }} s3 cp ./sample.txt s3://demo/sample.txt
|
||
</code></pre>
|
||
</div>
|
||
<div class="mb-4">
|
||
<h3 class="h6 text-uppercase text-muted">s3cmd</h3>
|
||
<pre class="mb-3"><code class="language-bash">cat > ~/.s3cfg-myfsio <<'EOF'
|
||
host_base = {{ api_host }}
|
||
host_bucket = %(bucket)s.{{ api_host }}
|
||
access_key = <access_key>
|
||
secret_key = <secret_key>
|
||
use_https = False
|
||
signature_v2 = False
|
||
EOF
|
||
|
||
s3cmd --config ~/.s3cfg-myfsio ls
|
||
s3cmd --config ~/.s3cfg-myfsio put notes.txt s3://demo/notes.txt
|
||
</code></pre>
|
||
</div>
|
||
<div>
|
||
<h3 class="h6 text-uppercase text-muted">curl / HTTPie</h3>
|
||
<pre class="mb-0"><code class="language-bash">curl {{ api_base }}/ \
|
||
-H "X-Access-Key: <access_key>" \
|
||
-H "X-Secret-Key: <secret_key>"
|
||
|
||
curl -X PUT {{ api_base }}/demo/notes.txt \
|
||
-H "X-Access-Key: <access_key>" \
|
||
-H "X-Secret-Key: <secret_key>" \
|
||
--data-binary @notes.txt
|
||
|
||
# Presigned URLs are generated via the UI
|
||
# Use the "Presign" button in the object browser
|
||
</code></pre>
|
||
</div>
|
||
</div>
|
||
</article>
|
||
<article id="api" class="card shadow-sm docs-section">
|
||
<div class="card-body">
|
||
<div class="d-flex align-items-center gap-2 mb-3">
|
||
<span class="docs-section-kicker">06</span>
|
||
<h2 class="h4 mb-0">Key REST endpoints</h2>
|
||
</div>
|
||
<div class="table-responsive">
|
||
<table class="table table-sm table-borderless align-middle docs-table mb-0">
|
||
<thead>
|
||
<tr>
|
||
<th scope="col">Method</th>
|
||
<th scope="col">Path</th>
|
||
<th scope="col">Purpose</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr>
|
||
<td>GET</td>
|
||
<td><code>/</code></td>
|
||
<td>List buckets accessible to the caller.</td>
|
||
</tr>
|
||
<tr>
|
||
<td>PUT</td>
|
||
<td><code>/<bucket></code></td>
|
||
<td>Create a bucket.</td>
|
||
</tr>
|
||
<tr>
|
||
<td>DELETE</td>
|
||
<td><code>/<bucket></code></td>
|
||
<td>Delete a bucket (must be empty).</td>
|
||
</tr>
|
||
<tr>
|
||
<td>GET</td>
|
||
<td><code>/<bucket></code></td>
|
||
<td>List objects (supports <code>prefix</code> / <code>max-keys</code> queries).</td>
|
||
</tr>
|
||
<tr>
|
||
<td>PUT</td>
|
||
<td><code>/<bucket>/<key></code></td>
|
||
<td>Upload or overwrite an object; UI helper handles multipart flows.</td>
|
||
</tr>
|
||
<tr>
|
||
<td>GET</td>
|
||
<td><code>/<bucket>/<key></code></td>
|
||
<td>Download an object (UI adds <code>?download=1</code> to force attachment).</td>
|
||
</tr>
|
||
<tr>
|
||
<td>DELETE</td>
|
||
<td><code>/<bucket>/<key></code></td>
|
||
<td>Delete an object.</td>
|
||
</tr>
|
||
<tr>
|
||
<td>HEAD</td>
|
||
<td><code>/<bucket></code></td>
|
||
<td>Check if a bucket exists.</td>
|
||
</tr>
|
||
<tr>
|
||
<td>HEAD</td>
|
||
<td><code>/<bucket>/<key></code></td>
|
||
<td>Get object metadata without downloading.</td>
|
||
</tr>
|
||
<tr>
|
||
<td>POST</td>
|
||
<td><code>/<bucket>?delete</code></td>
|
||
<td>Bulk delete objects (XML body).</td>
|
||
</tr>
|
||
<tr>
|
||
<td>GET/PUT/DELETE</td>
|
||
<td><code>/<bucket>?policy</code></td>
|
||
<td>Bucket policy management.</td>
|
||
</tr>
|
||
<tr>
|
||
<td>GET/PUT</td>
|
||
<td><code>/<bucket>?versioning</code></td>
|
||
<td>Versioning status.</td>
|
||
</tr>
|
||
<tr>
|
||
<td>GET/PUT/DELETE</td>
|
||
<td><code>/<bucket>?lifecycle</code></td>
|
||
<td>Lifecycle rules.</td>
|
||
</tr>
|
||
<tr>
|
||
<td>GET/PUT/DELETE</td>
|
||
<td><code>/<bucket>?cors</code></td>
|
||
<td>CORS configuration.</td>
|
||
</tr>
|
||
<tr>
|
||
<td>GET/PUT/DELETE</td>
|
||
<td><code>/<bucket>?encryption</code></td>
|
||
<td>Default encryption.</td>
|
||
</tr>
|
||
<tr>
|
||
<td>GET/PUT</td>
|
||
<td><code>/<bucket>?acl</code></td>
|
||
<td>Bucket ACL.</td>
|
||
</tr>
|
||
<tr>
|
||
<td>GET/PUT/DELETE</td>
|
||
<td><code>/<bucket>?tagging</code></td>
|
||
<td>Bucket tags.</td>
|
||
</tr>
|
||
<tr>
|
||
<td>GET/PUT/DELETE</td>
|
||
<td><code>/<bucket>/<key>?tagging</code></td>
|
||
<td>Object tags.</td>
|
||
</tr>
|
||
<tr>
|
||
<td>POST</td>
|
||
<td><code>/<bucket>/<key>?uploads</code></td>
|
||
<td>Initiate multipart upload.</td>
|
||
</tr>
|
||
<tr>
|
||
<td>POST</td>
|
||
<td><code>/<bucket>/<key>?select</code></td>
|
||
<td>SQL query (SelectObjectContent).</td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
</div>
|
||
<p class="small text-muted mt-3 mb-0">All responses include <code>X-Request-Id</code> for tracing. See the <a href="#api-matrix">Full API Reference</a> for the complete endpoint list. Logs land in <code>logs/api.log</code> and <code>logs/ui.log</code>.</p>
|
||
</div>
|
||
</article>
|
||
<article id="examples" class="card shadow-sm docs-section">
|
||
<div class="card-body">
|
||
<div class="d-flex align-items-center gap-2 mb-3">
|
||
<span class="docs-section-kicker">07</span>
|
||
<h2 class="h4 mb-0">API Examples</h2>
|
||
</div>
|
||
<p class="text-muted">Common operations using popular SDKs and tools.</p>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Python (boto3)</h3>
|
||
<pre class="mb-4"><code class="language-python">import boto3
|
||
|
||
s3 = boto3.client(
|
||
's3',
|
||
endpoint_url='{{ api_base }}',
|
||
aws_access_key_id='<access_key>',
|
||
aws_secret_access_key='<secret_key>'
|
||
)
|
||
|
||
# List buckets
|
||
buckets = s3.list_buckets()['Buckets']
|
||
|
||
# Create bucket
|
||
s3.create_bucket(Bucket='mybucket')
|
||
|
||
# Upload file
|
||
s3.upload_file('local.txt', 'mybucket', 'remote.txt')
|
||
|
||
# Download file
|
||
s3.download_file('mybucket', 'remote.txt', 'downloaded.txt')
|
||
|
||
# Generate presigned URL (valid 1 hour)
|
||
url = s3.generate_presigned_url(
|
||
'get_object',
|
||
Params={'Bucket': 'mybucket', 'Key': 'remote.txt'},
|
||
ExpiresIn=3600
|
||
)</code></pre>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">JavaScript (AWS SDK v3)</h3>
|
||
<pre class="mb-4"><code class="language-javascript">import { S3Client, ListBucketsCommand, PutObjectCommand } from '@aws-sdk/client-s3';
|
||
|
||
const s3 = new S3Client({
|
||
endpoint: '{{ api_base }}',
|
||
region: 'us-east-1',
|
||
credentials: {
|
||
accessKeyId: '<access_key>',
|
||
secretAccessKey: '<secret_key>'
|
||
},
|
||
forcePathStyle: true // Required for S3-compatible services
|
||
});
|
||
|
||
// List buckets
|
||
const { Buckets } = await s3.send(new ListBucketsCommand({}));
|
||
|
||
// Upload object
|
||
await s3.send(new PutObjectCommand({
|
||
Bucket: 'mybucket',
|
||
Key: 'hello.txt',
|
||
Body: 'Hello, World!'
|
||
}));</code></pre>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Multipart Upload (Python)</h3>
|
||
<pre class="mb-4"><code class="language-python">import boto3
|
||
|
||
s3 = boto3.client('s3', endpoint_url='{{ api_base }}')
|
||
|
||
# Initiate
|
||
response = s3.create_multipart_upload(Bucket='mybucket', Key='large.bin')
|
||
upload_id = response['UploadId']
|
||
|
||
# Upload parts (minimum 5MB each, except last part)
|
||
parts = []
|
||
chunks = [b'chunk1...', b'chunk2...']
|
||
for part_number, chunk in enumerate(chunks, start=1):
|
||
response = s3.upload_part(
|
||
Bucket='mybucket',
|
||
Key='large.bin',
|
||
PartNumber=part_number,
|
||
UploadId=upload_id,
|
||
Body=chunk
|
||
)
|
||
parts.append({'PartNumber': part_number, 'ETag': response['ETag']})
|
||
|
||
# Complete
|
||
s3.complete_multipart_upload(
|
||
Bucket='mybucket',
|
||
Key='large.bin',
|
||
UploadId=upload_id,
|
||
MultipartUpload={'Parts': parts}
|
||
)</code></pre>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Presigned URLs for Sharing</h3>
|
||
<pre class="mb-0"><code class="language-text"># Generate presigned URLs via the UI:
|
||
# 1. Navigate to your bucket in the object browser
|
||
# 2. Select the object you want to share
|
||
# 3. Click the "Presign" button
|
||
# 4. Choose method (GET/PUT/DELETE) and expiration time
|
||
# 5. Copy the generated URL
|
||
|
||
# Supported options:
|
||
# - Method: GET (download), PUT (upload), DELETE (remove)
|
||
# - Expiration: 1 second to 7 days (604800 seconds)</code></pre>
|
||
</div>
|
||
</article>
|
||
<article id="replication" class="card shadow-sm docs-section">
|
||
<div class="card-body">
|
||
<div class="d-flex align-items-center gap-2 mb-3">
|
||
<span class="docs-section-kicker">08</span>
|
||
<h2 class="h4 mb-0">Site Replication & Sync</h2>
|
||
</div>
|
||
<p class="text-muted">Replicate objects to another MyFSIO instance or S3-compatible service. Supports one-way replication for backup and bi-directional sync for geo-distributed deployments.</p>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Setup Guide</h3>
|
||
<ol class="docs-steps mb-3">
|
||
<li>
|
||
<strong>Prepare Target:</strong> On the destination server, create a bucket (e.g., <code>backup-bucket</code>) and an IAM user with write permissions.
|
||
</li>
|
||
<li>
|
||
<strong>Connect Source:</strong> On this server, go to <a href="{{ url_for('ui.connections_dashboard') }}">Connections</a> and add the target's API URL and credentials.
|
||
</li>
|
||
<li>
|
||
<strong>Enable Rule:</strong> Go to the source bucket's <strong>Replication</strong> tab, select the connection, and enter the target bucket name.
|
||
</li>
|
||
</ol>
|
||
|
||
<div class="alert alert-light border mb-3 overflow-hidden">
|
||
<div class="d-flex flex-column flex-sm-row gap-2 mb-2">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-terminal text-muted mt-1 flex-shrink-0 d-none d-sm-block" viewBox="0 0 16 16">
|
||
<path d="M6 9a.5.5 0 0 1 .5-.5h3a.5.5 0 0 1 0 1h-3A.5.5 0 0 1 6 9zM3.854 4.146a.5.5 0 1 0-.708.708L4.793 6.5 3.146 8.146a.5.5 0 1 0 .708.708l2-2a.5.5 0 0 0 0-.708l-2-2z"/>
|
||
<path d="M2 1a2 2 0 0 0-2 2v10a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V3a2 2 0 0 0-2-2H2zm12 1a1 1 0 0 1 1 1v10a1 1 0 0 1-1 1H2a1 1 0 0 1-1-1V3a1 1 0 0 1 1-1h12z"/>
|
||
</svg>
|
||
<div class="flex-grow-1 min-width-0">
|
||
<strong>Headless Target Setup</strong>
|
||
<p class="small text-muted mb-2">If your target server has no UI, create a <code>setup_target.py</code> script to bootstrap credentials:</p>
|
||
<pre class="mb-0 overflow-auto" style="max-width: 100%;"><code class="language-python"># setup_target.py
|
||
from pathlib import Path
|
||
from app.iam import IamService
|
||
from app.storage import ObjectStorage
|
||
|
||
# Initialize services (paths match default config)
|
||
data_dir = Path("data")
|
||
iam = IamService(data_dir / ".myfsio.sys" / "config" / "iam.json")
|
||
storage = ObjectStorage(data_dir)
|
||
|
||
# 1. Create the bucket
|
||
bucket_name = "backup-bucket"
|
||
try:
|
||
storage.create_bucket(bucket_name)
|
||
print(f"Bucket '{bucket_name}' created.")
|
||
except Exception as e:
|
||
print(f"Bucket creation skipped: {e}")
|
||
|
||
# 2. Create the user
|
||
try:
|
||
creds = iam.create_user(
|
||
display_name="Replication User",
|
||
policies=[{"bucket": bucket_name, "actions": ["write", "read", "list"]}]
|
||
)
|
||
print("\n--- CREDENTIALS GENERATED ---")
|
||
print(f"Access Key: {creds['access_key']}")
|
||
print(f"Secret Key: {creds['secret_key']}")
|
||
print("-----------------------------")
|
||
except Exception as e:
|
||
print(f"User creation failed: {e}")</code></pre>
|
||
<p class="small text-muted mt-2 mb-0">Save and run: <code>python setup_target.py</code></p>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Replication Modes</h3>
|
||
<div class="table-responsive mb-3">
|
||
<table class="table table-sm table-bordered small">
|
||
<thead class="table-light">
|
||
<tr>
|
||
<th>Mode</th>
|
||
<th>Description</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr>
|
||
<td><code>new_only</code></td>
|
||
<td>Only replicate new/modified objects (default, one-way)</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>all</code></td>
|
||
<td>Sync all existing objects when rule is enabled (one-way)</td>
|
||
</tr>
|
||
<tr>
|
||
<td><strong><code>bidirectional</code></strong></td>
|
||
<td>Two-way sync with Last-Write-Wins conflict resolution</td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
</div>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Bidirectional Site Replication</h3>
|
||
<p class="small text-muted">For true two-way synchronization with automatic conflict resolution, use the <code>bidirectional</code> mode. Both sites must be configured to sync with each other.</p>
|
||
|
||
<div class="alert alert-info border small mb-3">
|
||
<div class="d-flex gap-2">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-info-circle text-info mt-1 flex-shrink-0" viewBox="0 0 16 16">
|
||
<path d="M8 15A7 7 0 1 1 8 1a7 7 0 0 1 0 14zm0 1A8 8 0 1 0 8 0a8 8 0 0 0 0 16z"/>
|
||
<path d="m8.93 6.588-2.29.287-.082.38.45.083c.294.07.352.176.288.469l-.738 3.468c-.194.897.105 1.319.808 1.319.545 0 1.178-.252 1.465-.598l.088-.416c-.2.176-.492.246-.686.246-.275 0-.375-.193-.304-.533L8.93 6.588zM9 4.5a1 1 0 1 1-2 0 1 1 0 0 1 2 0z"/>
|
||
</svg>
|
||
<div>
|
||
<strong>Both sites need configuration.</strong> Each site pushes its changes and pulls from the other. You must set up connections and replication rules on both ends.
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<h4 class="h6 mt-4 mb-2">Step 1: Enable Site Sync on Both Sites</h4>
|
||
<p class="small text-muted">Set these environment variables on <strong>both</strong> Site A and Site B:</p>
|
||
<pre class="mb-3"><code class="language-bash">SITE_SYNC_ENABLED=true
|
||
SITE_SYNC_INTERVAL_SECONDS=60 # How often to pull changes
|
||
SITE_SYNC_BATCH_SIZE=100 # Max objects per sync cycle</code></pre>
|
||
|
||
<h4 class="h6 mt-4 mb-2">Step 2: Create IAM Users for Cross-Site Access</h4>
|
||
<p class="small text-muted">On each site, create an IAM user that the other site will use to connect:</p>
|
||
<div class="table-responsive mb-3">
|
||
<table class="table table-sm table-bordered small">
|
||
<thead class="table-light">
|
||
<tr>
|
||
<th>Site</th>
|
||
<th>Create User For</th>
|
||
<th>Required Permissions</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr>
|
||
<td>Site A</td>
|
||
<td>Site B to connect</td>
|
||
<td><code>read</code>, <code>write</code>, <code>list</code>, <code>delete</code> on target bucket</td>
|
||
</tr>
|
||
<tr>
|
||
<td>Site B</td>
|
||
<td>Site A to connect</td>
|
||
<td><code>read</code>, <code>write</code>, <code>list</code>, <code>delete</code> on target bucket</td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
</div>
|
||
|
||
<h4 class="h6 mt-4 mb-2">Step 3: Create Connections</h4>
|
||
<p class="small text-muted">On each site, add a connection pointing to the other:</p>
|
||
<div class="row g-3 mb-3">
|
||
<div class="col-md-6">
|
||
<div class="card border h-100">
|
||
<div class="card-header bg-light py-2"><strong class="small">On Site A</strong></div>
|
||
<div class="card-body small">
|
||
<p class="mb-1">Go to <strong>Connections</strong> and add:</p>
|
||
<ul class="mb-0 ps-3">
|
||
<li>Endpoint: <code>https://site-b.example.com</code></li>
|
||
<li>Credentials: Site B's IAM user</li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="col-md-6">
|
||
<div class="card border h-100">
|
||
<div class="card-header bg-light py-2"><strong class="small">On Site B</strong></div>
|
||
<div class="card-body small">
|
||
<p class="mb-1">Go to <strong>Connections</strong> and add:</p>
|
||
<ul class="mb-0 ps-3">
|
||
<li>Endpoint: <code>https://site-a.example.com</code></li>
|
||
<li>Credentials: Site A's IAM user</li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<h4 class="h6 mt-4 mb-2">Step 4: Enable Bidirectional Replication</h4>
|
||
<p class="small text-muted">On each site, go to the bucket's <strong>Replication</strong> tab and enable with mode <code>bidirectional</code>:</p>
|
||
<div class="row g-3 mb-3">
|
||
<div class="col-md-6">
|
||
<div class="card border h-100">
|
||
<div class="card-header bg-light py-2"><strong class="small">On Site A</strong></div>
|
||
<div class="card-body small">
|
||
<ul class="mb-0 ps-3">
|
||
<li>Source bucket: <code>my-bucket</code></li>
|
||
<li>Target: Site B connection</li>
|
||
<li>Target bucket: <code>my-bucket</code></li>
|
||
<li>Mode: <strong>Bidirectional sync</strong></li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="col-md-6">
|
||
<div class="card border h-100">
|
||
<div class="card-header bg-light py-2"><strong class="small">On Site B</strong></div>
|
||
<div class="card-body small">
|
||
<ul class="mb-0 ps-3">
|
||
<li>Source bucket: <code>my-bucket</code></li>
|
||
<li>Target: Site A connection</li>
|
||
<li>Target bucket: <code>my-bucket</code></li>
|
||
<li>Mode: <strong>Bidirectional sync</strong></li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<h4 class="h6 mt-4 mb-2">How It Works</h4>
|
||
<ul class="small text-muted mb-3">
|
||
<li><strong>PUSH:</strong> Local changes replicate to remote immediately on write/delete</li>
|
||
<li><strong>PULL:</strong> Background worker fetches remote changes every <code>SITE_SYNC_INTERVAL_SECONDS</code></li>
|
||
<li><strong>Conflict Resolution:</strong> Last-Write-Wins based on <code>last_modified</code> timestamps (1-second clock skew tolerance)</li>
|
||
<li><strong>Deletion Sync:</strong> Remote deletions propagate locally only for objects originally synced from remote</li>
|
||
<li><strong>Loop Prevention:</strong> <code>S3ReplicationAgent</code> and <code>SiteSyncAgent</code> User-Agents prevent infinite sync loops</li>
|
||
</ul>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Error Handling & Rate Limits</h3>
|
||
<p class="small text-muted mb-3">The replication system handles transient failures automatically:</p>
|
||
<div class="table-responsive mb-3">
|
||
<table class="table table-sm table-bordered small">
|
||
<thead class="table-light">
|
||
<tr>
|
||
<th>Behavior</th>
|
||
<th>Details</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr>
|
||
<td><strong>Retry Logic</strong></td>
|
||
<td>boto3 automatically handles 429 (rate limit) errors using exponential backoff with <code>max_attempts=2</code></td>
|
||
</tr>
|
||
<tr>
|
||
<td><strong>Concurrency</strong></td>
|
||
<td>Uses a ThreadPoolExecutor with 4 parallel workers for replication tasks</td>
|
||
</tr>
|
||
<tr>
|
||
<td><strong>Timeouts</strong></td>
|
||
<td>Connect: 5s, Read: 30s. Large files use streaming transfers</td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
</div>
|
||
<div class="alert alert-warning border mb-0">
|
||
<div class="d-flex gap-2">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-exclamation-triangle text-warning mt-1 flex-shrink-0" viewBox="0 0 16 16">
|
||
<path d="M7.938 2.016A.13.13 0 0 1 8.002 2a.13.13 0 0 1 .063.016.146.146 0 0 1 .054.057l6.857 11.667c.036.06.035.124.002.183a.163.163 0 0 1-.054.06.116.116 0 0 1-.066.017H1.146a.115.115 0 0 1-.066-.017.163.163 0 0 1-.054-.06.176.176 0 0 1 .002-.183L7.884 2.073a.147.147 0 0 1 .054-.057zm1.044-.45a1.13 1.13 0 0 0-1.96 0L.165 13.233c-.457.778.091 1.767.98 1.767h13.713c.889 0 1.438-.99.98-1.767L8.982 1.566z"/>
|
||
<path d="M7.002 12a1 1 0 1 1 2 0 1 1 0 0 1-2 0zM7.1 5.995a.905.905 0 1 1 1.8 0l-.35 3.507a.552.552 0 0 1-1.1 0L7.1 5.995z"/>
|
||
</svg>
|
||
<div>
|
||
<strong>Large File Counts:</strong> When replicating buckets with many objects, the target server's rate limits may cause delays. There is no built-in pause mechanism. Consider increasing <code>RATE_LIMIT_DEFAULT</code> on the target server during bulk replication operations.
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</article>
|
||
<article id="site-registry" class="card shadow-sm docs-section">
|
||
<div class="card-body">
|
||
<div class="d-flex align-items-center gap-2 mb-3">
|
||
<span class="docs-section-kicker">09</span>
|
||
<h2 class="h4 mb-0">Site Registry</h2>
|
||
</div>
|
||
<p class="text-muted">Track cluster membership and site identity for geo-distributed deployments. The site registry stores local site identity and peer site information.</p>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Connections vs Sites</h3>
|
||
<p class="small text-muted mb-3">Understanding the difference between Connections and Sites is key to configuring geo-distribution:</p>
|
||
<div class="table-responsive mb-3">
|
||
<table class="table table-sm table-bordered small">
|
||
<thead class="table-light">
|
||
<tr>
|
||
<th style="width: 20%;">Aspect</th>
|
||
<th style="width: 40%;">Connections</th>
|
||
<th style="width: 40%;">Sites</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr>
|
||
<td><strong>Purpose</strong></td>
|
||
<td>Store credentials to authenticate with remote S3 endpoints</td>
|
||
<td>Track cluster membership and site identity</td>
|
||
</tr>
|
||
<tr>
|
||
<td><strong>Contains</strong></td>
|
||
<td>Endpoint URL, access key, secret key, region</td>
|
||
<td>Site ID, endpoint, region, priority, display name</td>
|
||
</tr>
|
||
<tr>
|
||
<td><strong>Used by</strong></td>
|
||
<td>Replication rules, site sync workers</td>
|
||
<td>Geo-distribution awareness, cluster topology</td>
|
||
</tr>
|
||
<tr>
|
||
<td><strong>Analogy</strong></td>
|
||
<td><em>"How do I log in to that server?"</em></td>
|
||
<td><em>"Who are the members of my cluster?"</em></td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
</div>
|
||
<p class="small text-muted">Sites can optionally link to a Connection (via <code>connection_id</code>) to perform health checks against peer sites.</p>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Configuration</h3>
|
||
<p class="small text-muted">Set environment variables to bootstrap local site identity on startup:</p>
|
||
<div class="table-responsive mb-3">
|
||
<table class="table table-sm table-bordered small">
|
||
<thead class="table-light">
|
||
<tr>
|
||
<th>Variable</th>
|
||
<th>Default</th>
|
||
<th>Description</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr>
|
||
<td><code>SITE_ID</code></td>
|
||
<td><code>None</code></td>
|
||
<td>Unique identifier for this site (e.g., <code>us-west-1</code>)</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>SITE_ENDPOINT</code></td>
|
||
<td><code>None</code></td>
|
||
<td>Public URL for this site (e.g., <code>https://s3.us-west-1.example.com</code>)</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>SITE_REGION</code></td>
|
||
<td><code>us-east-1</code></td>
|
||
<td>AWS-style region identifier</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>SITE_PRIORITY</code></td>
|
||
<td><code>100</code></td>
|
||
<td>Routing priority (lower = preferred)</td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
</div>
|
||
<pre class="mb-3"><code class="language-bash"># Example: Configure site identity
|
||
export SITE_ID=us-west-1
|
||
export SITE_ENDPOINT=https://s3.us-west-1.example.com
|
||
export SITE_REGION=us-west-1
|
||
export SITE_PRIORITY=100
|
||
python run.py</code></pre>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Using the Sites UI</h3>
|
||
<p class="small text-muted">Navigate to <a href="{{ url_for('ui.sites_dashboard') }}">Sites</a> in the sidebar to manage site configuration:</p>
|
||
<div class="row g-3 mb-3">
|
||
<div class="col-md-6">
|
||
<div class="card border h-100">
|
||
<div class="card-header bg-light py-2"><strong class="small">Local Site Identity</strong></div>
|
||
<div class="card-body small">
|
||
<ul class="mb-0 ps-3">
|
||
<li>Configure this site's ID, endpoint, region, and priority</li>
|
||
<li>Display name for easier identification</li>
|
||
<li>Changes persist to <code>site_registry.json</code></li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="col-md-6">
|
||
<div class="card border h-100">
|
||
<div class="card-header bg-light py-2"><strong class="small">Peer Sites</strong></div>
|
||
<div class="card-body small">
|
||
<ul class="mb-0 ps-3">
|
||
<li>Register remote sites in your cluster</li>
|
||
<li>Link to a Connection for health checks</li>
|
||
<li>View health status (green/red/unknown)</li>
|
||
<li>Edit or delete peers as needed</li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Admin API Endpoints</h3>
|
||
<p class="small text-muted">The <code>/admin</code> API provides programmatic access to site registry:</p>
|
||
<pre class="mb-3"><code class="language-bash"># Get local site configuration
|
||
curl {{ api_base }}/admin/site \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"
|
||
|
||
# Update local site
|
||
curl -X PUT {{ api_base }}/admin/site \
|
||
-H "Content-Type: application/json" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||
-d '{"site_id": "us-west-1", "endpoint": "https://s3.example.com", "region": "us-west-1"}'
|
||
|
||
# List all peer sites
|
||
curl {{ api_base }}/admin/sites \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"
|
||
|
||
# Add a peer site
|
||
curl -X POST {{ api_base }}/admin/sites \
|
||
-H "Content-Type: application/json" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||
-d '{"site_id": "us-east-1", "endpoint": "https://s3.us-east-1.example.com"}'
|
||
|
||
# Check peer health
|
||
curl {{ api_base }}/admin/sites/us-east-1/health \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"
|
||
|
||
# Get cluster topology
|
||
curl {{ api_base }}/admin/topology \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"</code></pre>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Storage Location</h3>
|
||
<p class="small text-muted mb-3">Site registry data is stored at:</p>
|
||
<code class="d-block mb-3">data/.myfsio.sys/config/site_registry.json</code>
|
||
|
||
<div class="alert alert-light border mb-0">
|
||
<div class="d-flex gap-2">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-info-circle text-muted mt-1 flex-shrink-0" viewBox="0 0 16 16">
|
||
<path d="M8 15A7 7 0 1 1 8 1a7 7 0 0 1 0 14zm0 1A8 8 0 1 0 8 0a8 8 0 0 0 0 16z"/>
|
||
<path d="m8.93 6.588-2.29.287-.082.38.45.083c.294.07.352.176.288.469l-.738 3.468c-.194.897.105 1.319.808 1.319.545 0 1.178-.252 1.465-.598l.088-.416c-.2.176-.492.246-.686.246-.275 0-.375-.193-.304-.533L8.93 6.588zM9 4.5a1 1 0 1 1-2 0 1 1 0 0 1 2 0z"/>
|
||
</svg>
|
||
<div>
|
||
<strong>Planned:</strong> The site registry lays the groundwork for features like automatic failover, intelligent routing, and multi-site consistency. Currently it provides cluster awareness and health monitoring.
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</article>
|
||
<article id="versioning" class="card shadow-sm docs-section">
|
||
<div class="card-body">
|
||
<div class="d-flex align-items-center gap-2 mb-3">
|
||
<span class="docs-section-kicker">10</span>
|
||
<h2 class="h4 mb-0">Object Versioning</h2>
|
||
</div>
|
||
<p class="text-muted">Keep multiple versions of objects to protect against accidental deletions and overwrites. Restore previous versions at any time.</p>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Enabling Versioning</h3>
|
||
<ol class="docs-steps mb-3">
|
||
<li>Navigate to your bucket's <strong>Properties</strong> tab.</li>
|
||
<li>Find the <strong>Versioning</strong> card and click <strong>Enable</strong>.</li>
|
||
<li>All subsequent uploads will create new versions instead of overwriting.</li>
|
||
</ol>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Version Operations</h3>
|
||
<div class="table-responsive mb-3">
|
||
<table class="table table-sm table-bordered small">
|
||
<thead class="table-light">
|
||
<tr>
|
||
<th>Operation</th>
|
||
<th>Description</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr>
|
||
<td><strong>View Versions</strong></td>
|
||
<td>Click the version icon on any object to see all historical versions with timestamps and sizes.</td>
|
||
</tr>
|
||
<tr>
|
||
<td><strong>Restore Version</strong></td>
|
||
<td>Click <strong>Restore</strong> on any version to make it the current version (creates a copy).</td>
|
||
</tr>
|
||
<tr>
|
||
<td><strong>Delete Current</strong></td>
|
||
<td>Deleting an object archives it. Previous versions remain accessible.</td>
|
||
</tr>
|
||
<tr>
|
||
<td><strong>Purge All</strong></td>
|
||
<td>Permanently delete an object and all its versions. This cannot be undone.</td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
</div>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Archived Objects</h3>
|
||
<p class="small text-muted mb-3">When you delete a versioned object, it becomes "archived" - the current version is removed but historical versions remain. The <strong>Archived</strong> tab shows these objects so you can restore them.</p>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">API Usage</h3>
|
||
<pre class="mb-3"><code class="language-bash"># Enable versioning
|
||
curl -X PUT "{{ api_base }}/<bucket>?versioning" \
|
||
-H "Content-Type: application/json" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||
-d '{"Status": "Enabled"}'
|
||
|
||
# Get versioning status
|
||
curl "{{ api_base }}/<bucket>?versioning" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"
|
||
|
||
# List object versions
|
||
curl "{{ api_base }}/<bucket>?versions" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"
|
||
|
||
# Get specific version
|
||
curl "{{ api_base }}/<bucket>/<key>?versionId=<version-id>" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"</code></pre>
|
||
|
||
<div class="alert alert-light border mb-0">
|
||
<div class="d-flex gap-2">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-info-circle text-muted mt-1" viewBox="0 0 16 16">
|
||
<path d="M8 15A7 7 0 1 1 8 1a7 7 0 0 1 0 14zm0 1A8 8 0 1 0 8 0a8 8 0 0 0 0 16z"/>
|
||
<path d="m8.93 6.588-2.29.287-.082.38.45.083c.294.07.352.176.288.469l-.738 3.468c-.194.897.105 1.319.808 1.319.545 0 1.178-.252 1.465-.598l.088-.416c-.2.176-.492.246-.686.246-.275 0-.375-.193-.304-.533L8.93 6.588zM9 4.5a1 1 0 1 1-2 0 1 1 0 0 1 2 0z"/>
|
||
</svg>
|
||
<div>
|
||
<strong>Storage Impact:</strong> Each version consumes storage. Enable quotas to limit total bucket size including all versions.
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</article>
|
||
<article id="quotas" class="card shadow-sm docs-section">
|
||
<div class="card-body">
|
||
<div class="d-flex align-items-center gap-2 mb-3">
|
||
<span class="docs-section-kicker">11</span>
|
||
<h2 class="h4 mb-0">Bucket Quotas</h2>
|
||
</div>
|
||
<p class="text-muted">Limit how much data a bucket can hold using storage quotas. Quotas are enforced on uploads and multipart completions.</p>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Quota Types</h3>
|
||
<div class="table-responsive mb-3">
|
||
<table class="table table-sm table-bordered small">
|
||
<thead class="table-light">
|
||
<tr>
|
||
<th>Limit</th>
|
||
<th>Description</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr>
|
||
<td><strong>Max Size (MB)</strong></td>
|
||
<td>Maximum total storage in megabytes (includes current objects + archived versions)</td>
|
||
</tr>
|
||
<tr>
|
||
<td><strong>Max Objects</strong></td>
|
||
<td>Maximum number of objects (includes current objects + archived versions)</td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
</div>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Managing Quotas (Admin Only)</h3>
|
||
<p class="small text-muted">Quota management is restricted to administrators (users with <code>iam:*</code> permissions).</p>
|
||
<ol class="docs-steps mb-3">
|
||
<li>Navigate to your bucket → <strong>Properties</strong> tab → <strong>Storage Quota</strong> card.</li>
|
||
<li>Enter limits: <strong>Max Size (MB)</strong> and/or <strong>Max Objects</strong>. Leave empty for unlimited.</li>
|
||
<li>Click <strong>Update Quota</strong> to save, or <strong>Remove Quota</strong> to clear limits.</li>
|
||
</ol>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">API Usage</h3>
|
||
<pre class="mb-3"><code class="language-bash"># Set quota (max 100MB, max 1000 objects)
|
||
curl -X PUT "{{ api_base }}/bucket/<bucket>?quota" \
|
||
-H "Content-Type: application/json" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||
-d '{"max_bytes": 104857600, "max_objects": 1000}'
|
||
|
||
# Get current quota
|
||
curl "{{ api_base }}/bucket/<bucket>?quota" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"
|
||
|
||
# Remove quota
|
||
curl -X PUT "{{ api_base }}/bucket/<bucket>?quota" \
|
||
-H "Content-Type: application/json" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||
-d '{"max_bytes": null, "max_objects": null}'</code></pre>
|
||
|
||
<div class="alert alert-light border mb-0">
|
||
<div class="d-flex gap-2">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-info-circle text-muted mt-1" viewBox="0 0 16 16">
|
||
<path d="M8 15A7 7 0 1 1 8 1a7 7 0 0 1 0 14zm0 1A8 8 0 1 0 8 0a8 8 0 0 0 0 16z"/>
|
||
<path d="m8.93 6.588-2.29.287-.082.38.45.083c.294.07.352.176.288.469l-.738 3.468c-.194.897.105 1.319.808 1.319.545 0 1.178-.252 1.465-.598l.088-.416c-.2.176-.492.246-.686.246-.275 0-.375-.193-.304-.533L8.93 6.588zM9 4.5a1 1 0 1 1-2 0 1 1 0 0 1 2 0z"/>
|
||
</svg>
|
||
<div>
|
||
<strong>Version Counting:</strong> When versioning is enabled, archived versions count toward the quota. The quota is checked against total storage, not just current objects.
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</article>
|
||
<article id="encryption" class="card shadow-sm docs-section">
|
||
<div class="card-body">
|
||
<div class="d-flex align-items-center gap-2 mb-3">
|
||
<span class="docs-section-kicker">12</span>
|
||
<h2 class="h4 mb-0">Encryption</h2>
|
||
</div>
|
||
<p class="text-muted">Protect data at rest with server-side encryption using AES-256-GCM. Objects are encrypted before being written to disk and decrypted transparently on read.</p>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Encryption Types</h3>
|
||
<div class="table-responsive mb-3">
|
||
<table class="table table-sm table-bordered small">
|
||
<thead class="table-light">
|
||
<tr>
|
||
<th>Type</th>
|
||
<th>Description</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr>
|
||
<td><strong>AES-256 (SSE-S3)</strong></td>
|
||
<td>Server-managed encryption using a local master key</td>
|
||
</tr>
|
||
<tr>
|
||
<td><strong>KMS (SSE-KMS)</strong></td>
|
||
<td>Encryption using customer-managed keys via the built-in KMS</td>
|
||
</tr>
|
||
<tr>
|
||
<td><strong>SSE-C</strong></td>
|
||
<td>Server-side encryption with customer-provided keys (per-request)</td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
</div>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Enabling Encryption</h3>
|
||
<ol class="docs-steps mb-3">
|
||
<li>
|
||
<strong>Set environment variables:</strong>
|
||
<pre class="mb-2"><code class="language-bash"># PowerShell
|
||
$env:ENCRYPTION_ENABLED = "true"
|
||
$env:KMS_ENABLED = "true" # Optional
|
||
python run.py
|
||
|
||
# Bash
|
||
export ENCRYPTION_ENABLED=true
|
||
export KMS_ENABLED=true
|
||
python run.py</code></pre>
|
||
</li>
|
||
<li>
|
||
<strong>Configure bucket encryption:</strong> Navigate to your bucket → <strong>Properties</strong> tab → <strong>Default Encryption</strong> card → Click <strong>Enable Encryption</strong>.
|
||
</li>
|
||
<li>
|
||
<strong>Choose algorithm:</strong> Select <strong>AES-256</strong> for server-managed keys or <strong>aws:kms</strong> to use a KMS-managed key.
|
||
</li>
|
||
</ol>
|
||
|
||
<div class="alert alert-warning border-warning bg-warning-subtle mb-3">
|
||
<div class="d-flex gap-2">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-exclamation-triangle mt-1" viewBox="0 0 16 16">
|
||
<path d="M7.938 2.016A.13.13 0 0 1 8.002 2a.13.13 0 0 1 .063.016.146.146 0 0 1 .054.057l6.857 11.667c.036.06.035.124.002.183a.163.163 0 0 1-.054.06.116.116 0 0 1-.066.017H1.146a.115.115 0 0 1-.066-.017.163.163 0 0 1-.054-.06.176.176 0 0 1 .002-.183L7.884 2.073a.147.147 0 0 1 .054-.057zm1.044-.45a1.13 1.13 0 0 0-1.96 0L.165 13.233c-.457.778.091 1.767.98 1.767h13.713c.889 0 1.438-.99.98-1.767L8.982 1.566z"/>
|
||
<path d="M7.002 12a1 1 0 1 1 2 0 1 1 0 0 1-2 0zM7.1 5.995a.905.905 0 1 1 1.8 0l-.35 3.507a.552.552 0 0 1-1.1 0L7.1 5.995z"/>
|
||
</svg>
|
||
<div>
|
||
<strong>Important:</strong> Only <em>new uploads</em> after enabling encryption will be encrypted. Existing objects remain unencrypted.
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">KMS Key Management</h3>
|
||
<p class="small text-muted">When <code>KMS_ENABLED=true</code>, manage encryption keys via the API:</p>
|
||
<pre class="mb-3"><code class="language-bash"># Create a new KMS key
|
||
curl -X POST {{ api_base }}/kms/keys \
|
||
-H "Content-Type: application/json" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||
-d '{"alias": "my-key", "description": "Production key"}'
|
||
|
||
# List all keys
|
||
curl {{ api_base }}/kms/keys \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"
|
||
|
||
# Rotate a key (creates new key material)
|
||
curl -X POST {{ api_base }}/kms/keys/{key-id}/rotate \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"
|
||
|
||
# Disable/Enable a key
|
||
curl -X POST {{ api_base }}/kms/keys/{key-id}/disable \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"
|
||
|
||
# Schedule key deletion (30-day waiting period)
|
||
curl -X DELETE "{{ api_base }}/kms/keys/{key-id}?waiting_period_days=30" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"</code></pre>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">How It Works</h3>
|
||
<p class="small text-muted mb-0">
|
||
<strong>Envelope Encryption:</strong> Each object is encrypted with a unique Data Encryption Key (DEK). The DEK is then encrypted (wrapped) by the master key or KMS key and stored alongside the ciphertext. On read, the DEK is unwrapped and used to decrypt the object transparently.
|
||
</p>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">SSE-C (Customer-Provided Keys)</h3>
|
||
<p class="small text-muted">With SSE-C, you supply your own 256-bit AES key with each request. The server encrypts/decrypts using your key but never stores it. You must provide the same key for both upload and download.</p>
|
||
<div class="table-responsive mb-3">
|
||
<table class="table table-sm table-bordered small">
|
||
<thead class="table-light">
|
||
<tr>
|
||
<th>Header</th>
|
||
<th>Value</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr>
|
||
<td><code>x-amz-server-side-encryption-customer-algorithm</code></td>
|
||
<td><code>AES256</code></td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>x-amz-server-side-encryption-customer-key</code></td>
|
||
<td>Base64-encoded 256-bit key</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>x-amz-server-side-encryption-customer-key-MD5</code></td>
|
||
<td>Base64-encoded MD5 of the key</td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
</div>
|
||
<pre class="mb-3"><code class="language-bash"># Generate a 256-bit key
|
||
KEY=$(openssl rand -base64 32)
|
||
KEY_MD5=$(echo -n "$KEY" | base64 -d | openssl dgst -md5 -binary | base64)
|
||
|
||
# Upload with SSE-C
|
||
curl -X PUT "{{ api_base }}/my-bucket/secret.txt" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||
-H "x-amz-server-side-encryption-customer-algorithm: AES256" \
|
||
-H "x-amz-server-side-encryption-customer-key: $KEY" \
|
||
-H "x-amz-server-side-encryption-customer-key-MD5: $KEY_MD5" \
|
||
--data-binary @secret.txt
|
||
|
||
# Download with SSE-C (same key required)
|
||
curl "{{ api_base }}/my-bucket/secret.txt" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||
-H "x-amz-server-side-encryption-customer-algorithm: AES256" \
|
||
-H "x-amz-server-side-encryption-customer-key: $KEY" \
|
||
-H "x-amz-server-side-encryption-customer-key-MD5: $KEY_MD5"</code></pre>
|
||
<div class="alert alert-light border mb-0 small">
|
||
<strong>Note:</strong> SSE-C does not require <code>ENCRYPTION_ENABLED</code> or <code>KMS_ENABLED</code>. If you lose your key, the data is irrecoverable.
|
||
</div>
|
||
</div>
|
||
</article>
|
||
<article id="lifecycle" class="card shadow-sm docs-section">
|
||
<div class="card-body">
|
||
<div class="d-flex align-items-center gap-2 mb-3">
|
||
<span class="docs-section-kicker">13</span>
|
||
<h2 class="h4 mb-0">Lifecycle Rules</h2>
|
||
</div>
|
||
<p class="text-muted">Automatically delete expired objects, clean up old versions, and abort incomplete multipart uploads using time-based lifecycle rules.</p>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">How It Works</h3>
|
||
<p class="small text-muted mb-3">
|
||
Lifecycle rules run on a background timer (Python <code>threading.Timer</code>), not a system cronjob. The enforcement cycle triggers every <strong>3600 seconds (1 hour)</strong> by default. Each cycle scans all buckets with lifecycle configurations and applies matching rules.
|
||
</p>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Expiration Types</h3>
|
||
<div class="table-responsive mb-3">
|
||
<table class="table table-sm table-bordered small">
|
||
<thead class="table-light">
|
||
<tr>
|
||
<th>Type</th>
|
||
<th>Description</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr>
|
||
<td><strong>Expiration (Days)</strong></td>
|
||
<td>Delete current objects older than N days from their last modification</td>
|
||
</tr>
|
||
<tr>
|
||
<td><strong>Expiration (Date)</strong></td>
|
||
<td>Delete current objects after a specific date (ISO 8601 format)</td>
|
||
</tr>
|
||
<tr>
|
||
<td><strong>NoncurrentVersionExpiration</strong></td>
|
||
<td>Delete non-current (archived) versions older than N days from when they became non-current</td>
|
||
</tr>
|
||
<tr>
|
||
<td><strong>AbortIncompleteMultipartUpload</strong></td>
|
||
<td>Abort multipart uploads that have been in progress longer than N days</td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
</div>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">API Usage</h3>
|
||
<pre class="mb-3"><code class="language-bash"># Set lifecycle rule (delete objects older than 30 days)
|
||
curl -X PUT "{{ api_base }}/<bucket>?lifecycle" \
|
||
-H "Content-Type: application/json" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||
-d '[{
|
||
"ID": "expire-old-objects",
|
||
"Status": "Enabled",
|
||
"Prefix": "",
|
||
"Expiration": {"Days": 30}
|
||
}]'
|
||
|
||
# Abort incomplete multipart uploads after 7 days
|
||
curl -X PUT "{{ api_base }}/<bucket>?lifecycle" \
|
||
-H "Content-Type: application/json" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||
-d '[{
|
||
"ID": "cleanup-multipart",
|
||
"Status": "Enabled",
|
||
"AbortIncompleteMultipartUpload": {"DaysAfterInitiation": 7}
|
||
}]'
|
||
|
||
# Get current lifecycle configuration
|
||
curl "{{ api_base }}/<bucket>?lifecycle" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"</code></pre>
|
||
|
||
<div class="alert alert-light border mb-0">
|
||
<div class="d-flex gap-2">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-info-circle text-muted mt-1 flex-shrink-0" viewBox="0 0 16 16">
|
||
<path d="M8 15A7 7 0 1 1 8 1a7 7 0 0 1 0 14zm0 1A8 8 0 1 0 8 0a8 8 0 0 0 0 16z"/>
|
||
<path d="m8.93 6.588-2.29.287-.082.38.45.083c.294.07.352.176.288.469l-.738 3.468c-.194.897.105 1.319.808 1.319.545 0 1.178-.252 1.465-.598l.088-.416c-.2.176-.492.246-.686.246-.275 0-.375-.193-.304-.533L8.93 6.588zM9 4.5a1 1 0 1 1-2 0 1 1 0 0 1 2 0z"/>
|
||
</svg>
|
||
<div>
|
||
<strong>Prefix Filtering:</strong> Use the <code>Prefix</code> field to scope rules to specific paths (e.g., <code>"logs/"</code>). Leave empty to apply to all objects in the bucket.
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</article>
|
||
<article id="garbage-collection" class="card shadow-sm docs-section">
|
||
<div class="card-body">
|
||
<div class="d-flex align-items-center gap-2 mb-3">
|
||
<span class="docs-section-kicker">14</span>
|
||
<h2 class="h4 mb-0">Garbage Collection</h2>
|
||
</div>
|
||
<p class="text-muted">Automatically clean up orphaned data that accumulates over time: stale temp files, abandoned multipart uploads, stale lock files, orphaned metadata, orphaned versions, and empty directories.</p>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Enabling GC</h3>
|
||
<p class="small text-muted">Disabled by default. Enable via environment variable:</p>
|
||
<pre class="mb-3"><code class="language-bash">GC_ENABLED=true python run.py</code></pre>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Configuration</h3>
|
||
<div class="table-responsive mb-3">
|
||
<table class="table table-sm table-bordered small">
|
||
<thead class="table-light">
|
||
<tr>
|
||
<th>Variable</th>
|
||
<th>Default</th>
|
||
<th>Description</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr><td><code>GC_ENABLED</code></td><td><code>false</code></td><td>Enable garbage collection</td></tr>
|
||
<tr><td><code>GC_INTERVAL_HOURS</code></td><td><code>6</code></td><td>Hours between GC cycles</td></tr>
|
||
<tr><td><code>GC_TEMP_FILE_MAX_AGE_HOURS</code></td><td><code>24</code></td><td>Delete temp files older than this</td></tr>
|
||
<tr><td><code>GC_MULTIPART_MAX_AGE_DAYS</code></td><td><code>7</code></td><td>Delete orphaned multipart uploads older than this</td></tr>
|
||
<tr><td><code>GC_LOCK_FILE_MAX_AGE_HOURS</code></td><td><code>1</code></td><td>Delete stale lock files older than this</td></tr>
|
||
<tr><td><code>GC_DRY_RUN</code></td><td><code>false</code></td><td>Log what would be deleted without removing</td></tr>
|
||
</tbody>
|
||
</table>
|
||
</div>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">What Gets Cleaned</h3>
|
||
<div class="table-responsive mb-3">
|
||
<table class="table table-sm table-bordered small">
|
||
<thead class="table-light">
|
||
<tr>
|
||
<th>Type</th>
|
||
<th>Location</th>
|
||
<th>Condition</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr><td><strong>Temp files</strong></td><td><code>.myfsio.sys/tmp/</code></td><td>Older than configured max age</td></tr>
|
||
<tr><td><strong>Orphaned multipart</strong></td><td><code>.myfsio.sys/multipart/</code></td><td>Older than configured max age</td></tr>
|
||
<tr><td><strong>Stale lock files</strong></td><td><code>.myfsio.sys/buckets/<bucket>/locks/</code></td><td>Older than configured max age</td></tr>
|
||
<tr><td><strong>Orphaned metadata</strong></td><td><code>.myfsio.sys/buckets/<bucket>/meta/</code></td><td>Object file no longer exists</td></tr>
|
||
<tr><td><strong>Orphaned versions</strong></td><td><code>.myfsio.sys/buckets/<bucket>/versions/</code></td><td>Main object no longer exists</td></tr>
|
||
<tr><td><strong>Empty directories</strong></td><td>Various internal dirs</td><td>Directory is empty after cleanup</td></tr>
|
||
</tbody>
|
||
</table>
|
||
</div>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Admin API</h3>
|
||
<div class="table-responsive mb-3">
|
||
<table class="table table-sm table-bordered small">
|
||
<thead class="table-light">
|
||
<tr>
|
||
<th>Method</th>
|
||
<th>Route</th>
|
||
<th>Description</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr><td><code>GET</code></td><td><code>/admin/gc/status</code></td><td>Get GC status and configuration</td></tr>
|
||
<tr><td><code>POST</code></td><td><code>/admin/gc/run</code></td><td>Trigger manual GC run</td></tr>
|
||
<tr><td><code>GET</code></td><td><code>/admin/gc/history</code></td><td>Get execution history</td></tr>
|
||
</tbody>
|
||
</table>
|
||
</div>
|
||
|
||
<pre class="mb-3"><code class="language-bash"># Trigger a dry run (preview what would be cleaned)
|
||
curl -X POST "{{ api_base }}/admin/gc/run" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||
-H "Content-Type: application/json" \
|
||
-d '{"dry_run": true}'
|
||
|
||
# Trigger actual GC
|
||
curl -X POST "{{ api_base }}/admin/gc/run" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"
|
||
|
||
# Check status
|
||
curl "{{ api_base }}/admin/gc/status" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"
|
||
|
||
# View history
|
||
curl "{{ api_base }}/admin/gc/history?limit=10" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"</code></pre>
|
||
|
||
<div class="alert alert-light border mb-0">
|
||
<div class="d-flex gap-2">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-info-circle text-muted mt-1 flex-shrink-0" viewBox="0 0 16 16">
|
||
<path d="M8 15A7 7 0 1 1 8 1a7 7 0 0 1 0 14zm0 1A8 8 0 1 0 8 0a8 8 0 0 0 0 16z"/>
|
||
<path d="m8.93 6.588-2.29.287-.082.38.45.083c.294.07.352.176.288.469l-.738 3.468c-.194.897.105 1.319.808 1.319.545 0 1.178-.252 1.465-.598l.088-.416c-.2.176-.492.246-.686.246-.275 0-.375-.193-.304-.533L8.93 6.588zM9 4.5a1 1 0 1 1-2 0 1 1 0 0 1 2 0z"/>
|
||
</svg>
|
||
<div>
|
||
<strong>Dry Run:</strong> Use <code>GC_DRY_RUN=true</code> or pass <code>{"dry_run": true}</code> to the API to preview what would be deleted without actually removing anything. Check the logs or API response for details.
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</article>
|
||
<article id="integrity" class="card shadow-sm docs-section">
|
||
<div class="card-body">
|
||
<div class="d-flex align-items-center gap-2 mb-3">
|
||
<span class="docs-section-kicker">15</span>
|
||
<h2 class="h4 mb-0">Integrity Scanner</h2>
|
||
</div>
|
||
<p class="text-muted">Detect and optionally auto-repair data inconsistencies: corrupted objects, orphaned files, phantom metadata, stale versions, ETag cache drift, and unmigrated legacy metadata.</p>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Enabling Integrity Scanner</h3>
|
||
<p class="small text-muted">Disabled by default. Enable via environment variable:</p>
|
||
<pre class="mb-3"><code class="language-bash">INTEGRITY_ENABLED=true python run.py</code></pre>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Configuration</h3>
|
||
<div class="table-responsive mb-3">
|
||
<table class="table table-sm table-bordered small">
|
||
<thead class="table-light">
|
||
<tr>
|
||
<th>Variable</th>
|
||
<th>Default</th>
|
||
<th>Description</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr><td><code>INTEGRITY_ENABLED</code></td><td><code>false</code></td><td>Enable background integrity scanning</td></tr>
|
||
<tr><td><code>INTEGRITY_INTERVAL_HOURS</code></td><td><code>24</code></td><td>Hours between scan cycles</td></tr>
|
||
<tr><td><code>INTEGRITY_BATCH_SIZE</code></td><td><code>1000</code></td><td>Max objects to scan per cycle</td></tr>
|
||
<tr><td><code>INTEGRITY_AUTO_HEAL</code></td><td><code>false</code></td><td>Automatically repair detected issues</td></tr>
|
||
<tr><td><code>INTEGRITY_DRY_RUN</code></td><td><code>false</code></td><td>Log issues without healing</td></tr>
|
||
</tbody>
|
||
</table>
|
||
</div>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">What Gets Checked</h3>
|
||
<div class="table-responsive mb-3">
|
||
<table class="table table-sm table-bordered small">
|
||
<thead class="table-light">
|
||
<tr>
|
||
<th>Check</th>
|
||
<th>Detection</th>
|
||
<th>Heal Action</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr><td><strong>Corrupted objects</strong></td><td>File MD5 does not match stored ETag</td><td>Update ETag in index (disk is authoritative)</td></tr>
|
||
<tr><td><strong>Orphaned objects</strong></td><td>File exists without metadata entry</td><td>Create index entry with computed MD5/size/mtime</td></tr>
|
||
<tr><td><strong>Phantom metadata</strong></td><td>Index entry exists but file is missing</td><td>Remove stale entry from index</td></tr>
|
||
<tr><td><strong>Stale versions</strong></td><td>Manifest without data or vice versa</td><td>Remove orphaned version file</td></tr>
|
||
<tr><td><strong>ETag cache</strong></td><td><code>etag_index.json</code> differs from metadata</td><td>Delete cache file (auto-rebuilt)</td></tr>
|
||
<tr><td><strong>Legacy metadata</strong></td><td>Legacy <code>.meta.json</code> differs or unmigrated</td><td>Migrate to index, delete legacy file</td></tr>
|
||
</tbody>
|
||
</table>
|
||
</div>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Admin API</h3>
|
||
<div class="table-responsive mb-3">
|
||
<table class="table table-sm table-bordered small">
|
||
<thead class="table-light">
|
||
<tr>
|
||
<th>Method</th>
|
||
<th>Route</th>
|
||
<th>Description</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr><td><code>GET</code></td><td><code>/admin/integrity/status</code></td><td>Get scanner status and configuration</td></tr>
|
||
<tr><td><code>POST</code></td><td><code>/admin/integrity/run</code></td><td>Trigger manual scan</td></tr>
|
||
<tr><td><code>GET</code></td><td><code>/admin/integrity/history</code></td><td>Get scan history</td></tr>
|
||
</tbody>
|
||
</table>
|
||
</div>
|
||
|
||
<pre class="mb-3"><code class="language-bash"># Trigger a dry run with auto-heal preview
|
||
curl -X POST "{{ api_base }}/admin/integrity/run" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||
-H "Content-Type: application/json" \
|
||
-d '{"dry_run": true, "auto_heal": true}'
|
||
|
||
# Trigger actual scan with healing
|
||
curl -X POST "{{ api_base }}/admin/integrity/run" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||
-H "Content-Type: application/json" \
|
||
-d '{"auto_heal": true}'
|
||
|
||
# Check status
|
||
curl "{{ api_base }}/admin/integrity/status" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"
|
||
|
||
# View history
|
||
curl "{{ api_base }}/admin/integrity/history?limit=10" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"</code></pre>
|
||
|
||
<div class="alert alert-light border mb-0">
|
||
<div class="d-flex gap-2">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-info-circle text-muted mt-1 flex-shrink-0" viewBox="0 0 16 16">
|
||
<path d="M8 15A7 7 0 1 1 8 1a7 7 0 0 1 0 14zm0 1A8 8 0 1 0 8 0a8 8 0 0 0 0 16z"/>
|
||
<path d="m8.93 6.588-2.29.287-.082.38.45.083c.294.07.352.176.288.469l-.738 3.468c-.194.897.105 1.319.808 1.319.545 0 1.178-.252 1.465-.598l.088-.416c-.2.176-.492.246-.686.246-.275 0-.375-.193-.304-.533L8.93 6.588zM9 4.5a1 1 0 1 1-2 0 1 1 0 0 1 2 0z"/>
|
||
</svg>
|
||
<div>
|
||
<strong>Dry Run:</strong> Use <code>INTEGRITY_DRY_RUN=true</code> or pass <code>{"dry_run": true}</code> to the API to preview detected issues without making any changes. Combine with <code>{"auto_heal": true}</code> to see what would be repaired.
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</article>
|
||
<article id="metrics" class="card shadow-sm docs-section">
|
||
<div class="card-body">
|
||
<div class="d-flex align-items-center gap-2 mb-3">
|
||
<span class="docs-section-kicker">16</span>
|
||
<h2 class="h4 mb-0">Metrics History</h2>
|
||
</div>
|
||
<p class="text-muted">Track CPU, memory, and disk usage over time with optional metrics history. Disabled by default to minimize overhead.</p>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Enabling Metrics History</h3>
|
||
<p class="small text-muted">Set the environment variable to opt-in:</p>
|
||
<pre class="mb-3"><code class="language-bash"># PowerShell
|
||
$env:METRICS_HISTORY_ENABLED = "true"
|
||
python run.py
|
||
|
||
# Bash
|
||
export METRICS_HISTORY_ENABLED=true
|
||
python run.py</code></pre>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Configuration Options</h3>
|
||
<div class="table-responsive mb-3">
|
||
<table class="table table-sm table-bordered small">
|
||
<thead class="table-light">
|
||
<tr>
|
||
<th>Variable</th>
|
||
<th>Default</th>
|
||
<th>Description</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr>
|
||
<td><code>METRICS_HISTORY_ENABLED</code></td>
|
||
<td><code>false</code></td>
|
||
<td>Enable/disable metrics history recording</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>METRICS_HISTORY_RETENTION_HOURS</code></td>
|
||
<td><code>24</code></td>
|
||
<td>How long to keep history data (hours)</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>METRICS_HISTORY_INTERVAL_MINUTES</code></td>
|
||
<td><code>5</code></td>
|
||
<td>Interval between snapshots (minutes)</td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
</div>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">API Endpoints</h3>
|
||
<pre class="mb-3"><code class="language-bash"># Get metrics history (last 24 hours by default)
|
||
curl "{{ api_base | replace('/api', '/ui') }}/metrics/history" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"
|
||
|
||
# Get history for specific time range
|
||
curl "{{ api_base | replace('/api', '/ui') }}/metrics/history?hours=6" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"
|
||
|
||
# Get current settings
|
||
curl "{{ api_base | replace('/api', '/ui') }}/metrics/settings" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"
|
||
|
||
# Update settings at runtime
|
||
curl -X PUT "{{ api_base | replace('/api', '/ui') }}/metrics/settings" \
|
||
-H "Content-Type: application/json" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||
-d '{"enabled": true, "retention_hours": 48, "interval_minutes": 10}'</code></pre>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Storage Location</h3>
|
||
<p class="small text-muted mb-3">History data is stored at:</p>
|
||
<code class="d-block mb-3">data/.myfsio.sys/config/metrics_history.json</code>
|
||
|
||
<div class="alert alert-light border mb-0">
|
||
<div class="d-flex gap-2">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-info-circle text-muted mt-1 flex-shrink-0" viewBox="0 0 16 16">
|
||
<path d="M8 15A7 7 0 1 1 8 1a7 7 0 0 1 0 14zm0 1A8 8 0 1 0 8 0a8 8 0 0 0 0 16z"/>
|
||
<path d="m8.93 6.588-2.29.287-.082.38.45.083c.294.07.352.176.288.469l-.738 3.468c-.194.897.105 1.319.808 1.319.545 0 1.178-.252 1.465-.598l.088-.416c-.2.176-.492.246-.686.246-.275 0-.375-.193-.304-.533L8.93 6.588zM9 4.5a1 1 0 1 1-2 0 1 1 0 0 1 2 0z"/>
|
||
</svg>
|
||
<div>
|
||
<strong>UI Charts:</strong> When enabled, the Metrics dashboard displays line charts showing CPU, memory, and disk usage trends with a time range selector (1h, 6h, 24h, 7d).
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</article>
|
||
<article id="operation-metrics" class="card shadow-sm docs-section">
|
||
<div class="card-body">
|
||
<div class="d-flex align-items-center gap-2 mb-3">
|
||
<span class="docs-section-kicker">17</span>
|
||
<h2 class="h4 mb-0">Operation Metrics</h2>
|
||
</div>
|
||
<p class="text-muted">Track API request statistics including request counts, latency, error rates, and bandwidth usage. Provides real-time visibility into API operations.</p>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Enabling Operation Metrics</h3>
|
||
<p class="small text-muted">Set the environment variable to opt-in:</p>
|
||
<pre class="mb-3"><code class="language-bash"># PowerShell
|
||
$env:OPERATION_METRICS_ENABLED = "true"
|
||
python run.py
|
||
|
||
# Bash
|
||
export OPERATION_METRICS_ENABLED=true
|
||
python run.py</code></pre>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Configuration Options</h3>
|
||
<div class="table-responsive mb-3">
|
||
<table class="table table-sm table-bordered small">
|
||
<thead class="table-light">
|
||
<tr>
|
||
<th>Variable</th>
|
||
<th>Default</th>
|
||
<th>Description</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr>
|
||
<td><code>OPERATION_METRICS_ENABLED</code></td>
|
||
<td><code>false</code></td>
|
||
<td>Enable/disable operation metrics collection</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>OPERATION_METRICS_INTERVAL_MINUTES</code></td>
|
||
<td><code>5</code></td>
|
||
<td>Interval between snapshots (minutes)</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>OPERATION_METRICS_RETENTION_HOURS</code></td>
|
||
<td><code>24</code></td>
|
||
<td>How long to keep history data (hours)</td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
</div>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">What's Tracked</h3>
|
||
<div class="row g-3 mb-4">
|
||
<div class="col-md-6">
|
||
<div class="bg-light rounded p-3 h-100">
|
||
<h6 class="small fw-bold mb-2">Request Statistics</h6>
|
||
<ul class="small text-muted mb-0 ps-3">
|
||
<li>Request counts by HTTP method (GET, PUT, POST, DELETE)</li>
|
||
<li>Response status codes (2xx, 3xx, 4xx, 5xx)</li>
|
||
<li>Average, min, max latency</li>
|
||
<li>Bytes transferred in/out</li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
<div class="col-md-6">
|
||
<div class="bg-light rounded p-3 h-100">
|
||
<h6 class="small fw-bold mb-2">Endpoint Breakdown</h6>
|
||
<ul class="small text-muted mb-0 ps-3">
|
||
<li><code>object</code> - Object operations (GET/PUT/DELETE)</li>
|
||
<li><code>bucket</code> - Bucket operations</li>
|
||
<li><code>ui</code> - Web UI requests</li>
|
||
<li><code>service</code> - Health checks, etc.</li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">S3 Error Codes</h3>
|
||
<p class="small text-muted">The dashboard tracks S3 API-specific error codes like <code>NoSuchKey</code>, <code>AccessDenied</code>, <code>BucketNotFound</code>. These are separate from HTTP status codes – a 404 from the UI won't appear here, only S3 API errors.</p>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">API Endpoints</h3>
|
||
<pre class="mb-3"><code class="language-bash"># Get current operation metrics
|
||
curl "{{ api_base | replace('/api', '/ui') }}/metrics/operations" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"
|
||
|
||
# Get operation metrics history
|
||
curl "{{ api_base | replace('/api', '/ui') }}/metrics/operations/history" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"
|
||
|
||
# Filter history by time range
|
||
curl "{{ api_base | replace('/api', '/ui') }}/metrics/operations/history?hours=6" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"</code></pre>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Storage Location</h3>
|
||
<p class="small text-muted mb-3">Operation metrics data is stored at:</p>
|
||
<code class="d-block mb-3">data/.myfsio.sys/config/operation_metrics.json</code>
|
||
|
||
<div class="alert alert-light border mb-0">
|
||
<div class="d-flex gap-2">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-info-circle text-muted mt-1 flex-shrink-0" viewBox="0 0 16 16">
|
||
<path d="M8 15A7 7 0 1 1 8 1a7 7 0 0 1 0 14zm0 1A8 8 0 1 0 8 0a8 8 0 0 0 0 16z"/>
|
||
<path d="m8.93 6.588-2.29.287-.082.38.45.083c.294.07.352.176.288.469l-.738 3.468c-.194.897.105 1.319.808 1.319.545 0 1.178-.252 1.465-.598l.088-.416c-.2.176-.492.246-.686.246-.275 0-.375-.193-.304-.533L8.93 6.588zM9 4.5a1 1 0 1 1-2 0 1 1 0 0 1 2 0z"/>
|
||
</svg>
|
||
<div>
|
||
<strong>UI Dashboard:</strong> When enabled, the Metrics page shows an "API Operations" section with summary cards, charts for requests by method/status/endpoint, and an S3 error codes table. Data refreshes every 5 seconds.
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</article>
|
||
<article id="troubleshooting" class="card shadow-sm docs-section">
|
||
<div class="card-body">
|
||
<div class="d-flex align-items-center gap-2 mb-3">
|
||
<span class="docs-section-kicker">18</span>
|
||
<h2 class="h4 mb-0">Troubleshooting & tips</h2>
|
||
</div>
|
||
<div class="table-responsive">
|
||
<table class="table table-sm align-middle docs-table mb-0">
|
||
<thead>
|
||
<tr>
|
||
<th scope="col">Symptom</th>
|
||
<th scope="col">Likely cause</th>
|
||
<th scope="col">Fix</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr>
|
||
<td>403 from API despite Public preset</td>
|
||
<td>Policy not saved or ARN mismatch</td>
|
||
<td>Reapply the preset and confirm <code>arn:aws:s3:::bucket/*</code> matches the bucket name.</td>
|
||
</tr>
|
||
<tr>
|
||
<td>UI shows stale policy/object data</td>
|
||
<td>Browser cached prior state</td>
|
||
<td>Refresh; the server hot-reloads <code>data/.myfsio.sys/config/bucket_policies.json</code> and storage metadata.</td>
|
||
</tr>
|
||
<tr>
|
||
<td>Presign dialog returns 403</td>
|
||
<td>User lacks required <code>read/write/delete</code> action or bucket policy denies</td>
|
||
<td>Update IAM inline policies or remove conflicting deny statements.</td>
|
||
</tr>
|
||
<tr>
|
||
<td>Large uploads fail instantly</td>
|
||
<td><code>MAX_UPLOAD_SIZE</code> exceeded</td>
|
||
<td>Raise the env var or split the object.</td>
|
||
</tr>
|
||
<tr>
|
||
<td>Requests hit the wrong host</td>
|
||
<td>Proxy headers missing or <code>API_BASE_URL</code> incorrect</td>
|
||
<td>Ensure your proxy sends <code>X-Forwarded-Host</code>/<code>Proto</code> headers, or explicitly set <code>API_BASE_URL</code> to your public domain.</td>
|
||
</tr>
|
||
<tr>
|
||
<td>Large folder uploads hitting rate limits (429)</td>
|
||
<td><code>RATE_LIMIT_DEFAULT</code> exceeded (200/min)</td>
|
||
<td>Increase rate limit in env config, use Redis backend (<code>RATE_LIMIT_STORAGE_URI=redis://host:port</code>) for distributed setups, or upload in smaller batches.</td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
</div>
|
||
</div>
|
||
</article>
|
||
<article id="health-check" class="card shadow-sm docs-section">
|
||
<div class="card-body">
|
||
<div class="d-flex align-items-center gap-2 mb-3">
|
||
<span class="docs-section-kicker">19</span>
|
||
<h2 class="h4 mb-0">Health Check Endpoint</h2>
|
||
</div>
|
||
<p class="text-muted">The API exposes a health check endpoint for monitoring and load balancer integration.</p>
|
||
|
||
<pre class="mb-3"><code class="language-bash"># Check API health
|
||
curl {{ api_base }}/myfsio/health
|
||
|
||
# Response
|
||
{"status": "ok", "version": "0.1.7"}</code></pre>
|
||
|
||
<p class="small text-muted mb-3">Use this endpoint for:</p>
|
||
<ul class="small text-muted mb-0">
|
||
<li>Load balancer health checks</li>
|
||
<li>Kubernetes liveness/readiness probes</li>
|
||
<li>Monitoring system integration (Prometheus, Datadog, etc.)</li>
|
||
</ul>
|
||
</div>
|
||
</article>
|
||
<article id="object-lock" class="card shadow-sm docs-section">
|
||
<div class="card-body">
|
||
<div class="d-flex align-items-center gap-2 mb-3">
|
||
<span class="docs-section-kicker">20</span>
|
||
<h2 class="h4 mb-0">Object Lock & Retention</h2>
|
||
</div>
|
||
<p class="text-muted">Object Lock prevents objects from being deleted or overwritten for a specified retention period.</p>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Retention Modes</h3>
|
||
<div class="table-responsive mb-3">
|
||
<table class="table table-sm table-bordered small">
|
||
<thead class="table-light">
|
||
<tr>
|
||
<th>Mode</th>
|
||
<th>Description</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr>
|
||
<td><strong>GOVERNANCE</strong></td>
|
||
<td>Objects can't be deleted by normal users, but admins with bypass permission can override</td>
|
||
</tr>
|
||
<tr>
|
||
<td><strong>COMPLIANCE</strong></td>
|
||
<td>Objects can't be deleted or overwritten by anyone until the retention period expires</td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
</div>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">API Usage</h3>
|
||
<pre class="mb-3"><code class="language-bash"># Set object retention
|
||
curl -X PUT "{{ api_base }}/<bucket>/<key>?retention" \
|
||
-H "Content-Type: application/json" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||
-d '{"Mode": "GOVERNANCE", "RetainUntilDate": "2025-12-31T23:59:59Z"}'
|
||
|
||
# Enable legal hold (indefinite protection)
|
||
curl -X PUT "{{ api_base }}/<bucket>/<key>?legal-hold" \
|
||
-H "Content-Type: application/json" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||
-d '{"Status": "ON"}'
|
||
|
||
# Get legal hold status
|
||
curl "{{ api_base }}/<bucket>/<key>?legal-hold" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"</code></pre>
|
||
|
||
<div class="alert alert-light border mb-0">
|
||
<div class="d-flex gap-2">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-info-circle text-muted mt-1 flex-shrink-0" viewBox="0 0 16 16">
|
||
<path d="M8 15A7 7 0 1 1 8 1a7 7 0 0 1 0 14zm0 1A8 8 0 1 0 8 0a8 8 0 0 0 0 16z"/>
|
||
<path d="m8.93 6.588-2.29.287-.082.38.45.083c.294.07.352.176.288.469l-.738 3.468c-.194.897.105 1.319.808 1.319.545 0 1.178-.252 1.465-.598l.088-.416c-.2.176-.492.246-.686.246-.275 0-.375-.193-.304-.533L8.93 6.588zM9 4.5a1 1 0 1 1-2 0 1 1 0 0 1 2 0z"/>
|
||
</svg>
|
||
<div>
|
||
<strong>Legal Hold:</strong> Provides indefinite protection independent of retention settings. Use for litigation holds or regulatory requirements.
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</article>
|
||
<article id="access-logging" class="card shadow-sm docs-section">
|
||
<div class="card-body">
|
||
<div class="d-flex align-items-center gap-2 mb-3">
|
||
<span class="docs-section-kicker">21</span>
|
||
<h2 class="h4 mb-0">Access Logging</h2>
|
||
</div>
|
||
<p class="text-muted">Enable S3-style access logging to track all requests to your buckets for audit and analysis.</p>
|
||
|
||
<pre class="mb-3"><code class="language-bash"># Enable access logging
|
||
curl -X PUT "{{ api_base }}/<bucket>?logging" \
|
||
-H "Content-Type: application/json" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||
-d '{
|
||
"LoggingEnabled": {
|
||
"TargetBucket": "log-bucket",
|
||
"TargetPrefix": "logs/my-bucket/"
|
||
}
|
||
}'
|
||
|
||
# Get logging configuration
|
||
curl "{{ api_base }}/<bucket>?logging" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"</code></pre>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Log Contents</h3>
|
||
<p class="small text-muted mb-0">Logs include: timestamp, bucket, key, operation type, request ID, requester, source IP, HTTP status, error codes, bytes transferred, timing, referrer, and User-Agent.</p>
|
||
</div>
|
||
</article>
|
||
<article id="notifications" class="card shadow-sm docs-section">
|
||
<div class="card-body">
|
||
<div class="d-flex align-items-center gap-2 mb-3">
|
||
<span class="docs-section-kicker">22</span>
|
||
<h2 class="h4 mb-0">Notifications & Webhooks</h2>
|
||
</div>
|
||
<p class="text-muted">Configure event notifications to trigger webhooks when objects are created or deleted.</p>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Supported Events</h3>
|
||
<div class="table-responsive mb-3">
|
||
<table class="table table-sm table-bordered small">
|
||
<thead class="table-light">
|
||
<tr>
|
||
<th>Event Type</th>
|
||
<th>Description</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr>
|
||
<td><code>s3:ObjectCreated:*</code></td>
|
||
<td>Any object creation (PUT, POST, COPY, multipart)</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>s3:ObjectRemoved:*</code></td>
|
||
<td>Any object deletion</td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
</div>
|
||
|
||
<pre class="mb-3"><code class="language-bash"># Set notification configuration
|
||
curl -X PUT "{{ api_base }}/<bucket>?notification" \
|
||
-H "Content-Type: application/json" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||
-d '{
|
||
"TopicConfigurations": [{
|
||
"Id": "upload-notify",
|
||
"TopicArn": "https://webhook.example.com/s3-events",
|
||
"Events": ["s3:ObjectCreated:*"],
|
||
"Filter": {
|
||
"Key": {
|
||
"FilterRules": [
|
||
{"Name": "prefix", "Value": "uploads/"},
|
||
{"Name": "suffix", "Value": ".jpg"}
|
||
]
|
||
}
|
||
}
|
||
}]
|
||
}'</code></pre>
|
||
|
||
<div class="alert alert-warning border-warning bg-warning-subtle mb-0">
|
||
<div class="d-flex gap-2">
|
||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-exclamation-triangle mt-1" viewBox="0 0 16 16">
|
||
<path d="M7.938 2.016A.13.13 0 0 1 8.002 2a.13.13 0 0 1 .063.016.146.146 0 0 1 .054.057l6.857 11.667c.036.06.035.124.002.183a.163.163 0 0 1-.054.06.116.116 0 0 1-.066.017H1.146a.115.115 0 0 1-.066-.017.163.163 0 0 1-.054-.06.176.176 0 0 1 .002-.183L7.884 2.073a.147.147 0 0 1 .054-.057zm1.044-.45a1.13 1.13 0 0 0-1.96 0L.165 13.233c-.457.778.091 1.767.98 1.767h13.713c.889 0 1.438-.99.98-1.767L8.982 1.566z"/>
|
||
<path d="M7.002 12a1 1 0 1 1 2 0 1 1 0 0 1-2 0zM7.1 5.995a.905.905 0 1 1 1.8 0l-.35 3.507a.552.552 0 0 1-1.1 0L7.1 5.995z"/>
|
||
</svg>
|
||
<div>
|
||
<strong>Security:</strong> Webhook URLs are validated to prevent SSRF attacks. Internal/private IP ranges are blocked.
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</article>
|
||
<article id="select-content" class="card shadow-sm docs-section">
|
||
<div class="card-body">
|
||
<div class="d-flex align-items-center gap-2 mb-3">
|
||
<span class="docs-section-kicker">23</span>
|
||
<h2 class="h4 mb-0">SelectObjectContent (SQL)</h2>
|
||
</div>
|
||
<p class="text-muted">Query CSV, JSON, or Parquet files directly using SQL without downloading the entire object.</p>
|
||
|
||
<div class="alert alert-info border small mb-3">
|
||
<strong>Prerequisite:</strong> Requires DuckDB to be installed (<code>pip install duckdb</code>)
|
||
</div>
|
||
|
||
<pre class="mb-3"><code class="language-bash"># Query a CSV file
|
||
curl -X POST "{{ api_base }}/<bucket>/data.csv?select" \
|
||
-H "Content-Type: application/json" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||
-d '{
|
||
"Expression": "SELECT name, age FROM s3object WHERE age > 25",
|
||
"ExpressionType": "SQL",
|
||
"InputSerialization": {
|
||
"CSV": {"FileHeaderInfo": "USE", "FieldDelimiter": ","}
|
||
},
|
||
"OutputSerialization": {"JSON": {}}
|
||
}'</code></pre>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Supported Formats</h3>
|
||
<div class="row g-2 mb-0">
|
||
<div class="col-md-4">
|
||
<div class="bg-light rounded p-2 small text-center">
|
||
<strong>CSV</strong><br><span class="text-muted">Headers, delimiters</span>
|
||
</div>
|
||
</div>
|
||
<div class="col-md-4">
|
||
<div class="bg-light rounded p-2 small text-center">
|
||
<strong>JSON</strong><br><span class="text-muted">Document or lines</span>
|
||
</div>
|
||
</div>
|
||
<div class="col-md-4">
|
||
<div class="bg-light rounded p-2 small text-center">
|
||
<strong>Parquet</strong><br><span class="text-muted">Auto schema</span>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</article>
|
||
<article id="advanced-ops" class="card shadow-sm docs-section">
|
||
<div class="card-body">
|
||
<div class="d-flex align-items-center gap-2 mb-3">
|
||
<span class="docs-section-kicker">24</span>
|
||
<h2 class="h4 mb-0">Advanced S3 Operations</h2>
|
||
</div>
|
||
<p class="text-muted">Copy, move, and partially download objects using advanced S3 operations.</p>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">CopyObject</h3>
|
||
<pre class="mb-3"><code class="language-bash"># Copy within same bucket
|
||
curl -X PUT "{{ api_base }}/<bucket>/copy-of-file.txt" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||
-H "x-amz-copy-source: /<bucket>/original-file.txt"
|
||
|
||
# Copy with metadata replacement
|
||
curl -X PUT "{{ api_base }}/<bucket>/file.txt" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||
-H "x-amz-copy-source: /<bucket>/file.txt" \
|
||
-H "x-amz-metadata-directive: REPLACE" \
|
||
-H "x-amz-meta-newkey: newvalue"</code></pre>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">MoveObject (UI)</h3>
|
||
<p class="small text-muted">Move an object to a different key or bucket via the UI. Performs a copy then deletes the source. Requires <code>read</code>+<code>delete</code> on source and <code>write</code> on destination.</p>
|
||
<pre class="mb-3"><code class="language-bash"># Move via UI API (session-authenticated)
|
||
curl -X POST "http://localhost:5100/ui/buckets/<bucket>/objects/<key>/move" \
|
||
-H "Content-Type: application/json" --cookie "session=..." \
|
||
-d '{"dest_bucket": "other-bucket", "dest_key": "new-path/file.txt"}'</code></pre>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">UploadPartCopy</h3>
|
||
<p class="small text-muted">Copy data from an existing object into a multipart upload part:</p>
|
||
<pre class="mb-3"><code class="language-bash"># Copy bytes 0-10485759 from source as part 1
|
||
curl -X PUT "{{ api_base }}/<bucket>/<key>?uploadId=X&partNumber=1" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||
-H "x-amz-copy-source: /source-bucket/source-file.bin" \
|
||
-H "x-amz-copy-source-range: bytes=0-10485759"</code></pre>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Range Requests</h3>
|
||
<pre class="mb-3"><code class="language-bash"># Get first 1000 bytes
|
||
curl "{{ api_base }}/<bucket>/<key>" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||
-H "Range: bytes=0-999"
|
||
|
||
# Get last 500 bytes
|
||
curl "{{ api_base }}/<bucket>/<key>" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||
-H "Range: bytes=-500"</code></pre>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Conditional Requests</h3>
|
||
<div class="table-responsive mb-0">
|
||
<table class="table table-sm table-bordered small mb-0">
|
||
<thead class="table-light">
|
||
<tr>
|
||
<th>Header</th>
|
||
<th>Behavior</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr>
|
||
<td><code>If-Modified-Since</code></td>
|
||
<td>Only download if changed after date</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>If-None-Match</code></td>
|
||
<td>Only download if ETag differs</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>If-Match</code></td>
|
||
<td>Only download if ETag matches</td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
</div>
|
||
</div>
|
||
</article>
|
||
<article id="acls" class="card shadow-sm docs-section">
|
||
<div class="card-body">
|
||
<div class="d-flex align-items-center gap-2 mb-3">
|
||
<span class="docs-section-kicker">25</span>
|
||
<h2 class="h4 mb-0">Access Control Lists (ACLs)</h2>
|
||
</div>
|
||
<p class="text-muted">ACLs provide legacy-style permission management for buckets and objects.</p>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Canned ACLs</h3>
|
||
<div class="table-responsive mb-3">
|
||
<table class="table table-sm table-bordered small">
|
||
<thead class="table-light">
|
||
<tr>
|
||
<th>ACL</th>
|
||
<th>Description</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr>
|
||
<td><code>private</code></td>
|
||
<td>Owner gets FULL_CONTROL (default)</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>public-read</code></td>
|
||
<td>Owner FULL_CONTROL, public READ</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>public-read-write</code></td>
|
||
<td>Owner FULL_CONTROL, public READ and WRITE</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>authenticated-read</code></td>
|
||
<td>Owner FULL_CONTROL, authenticated users READ</td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
</div>
|
||
|
||
<pre class="mb-3"><code class="language-bash"># Set bucket ACL
|
||
curl -X PUT "{{ api_base }}/<bucket>?acl" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||
-H "x-amz-acl: public-read"
|
||
|
||
# Set object ACL during upload
|
||
curl -X PUT "{{ api_base }}/<bucket>/<key>" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||
-H "x-amz-acl: private" \
|
||
--data-binary @file.txt</code></pre>
|
||
|
||
<div class="alert alert-light border mb-0">
|
||
<strong>Recommendation:</strong> For most use cases, prefer bucket policies over ACLs for more flexible access control.
|
||
</div>
|
||
</div>
|
||
</article>
|
||
<article id="tagging" class="card shadow-sm docs-section">
|
||
<div class="card-body">
|
||
<div class="d-flex align-items-center gap-2 mb-3">
|
||
<span class="docs-section-kicker">26</span>
|
||
<h2 class="h4 mb-0">Object & Bucket Tagging</h2>
|
||
</div>
|
||
<p class="text-muted">Add metadata tags to buckets and objects for organization, cost allocation, or lifecycle rule filtering.</p>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Object Tagging</h3>
|
||
<pre class="mb-3"><code class="language-bash"># Set object tags
|
||
curl -X PUT "{{ api_base }}/<bucket>/<key>?tagging" \
|
||
-H "Content-Type: application/json" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||
-d '{
|
||
"TagSet": [
|
||
{"Key": "Classification", "Value": "Confidential"},
|
||
{"Key": "Owner", "Value": "john@example.com"}
|
||
]
|
||
}'
|
||
|
||
# Get object tags
|
||
curl "{{ api_base }}/<bucket>/<key>?tagging" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"
|
||
|
||
# Set tags during upload
|
||
curl -X PUT "{{ api_base }}/<bucket>/<key>" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||
-H "x-amz-tagging: Environment=Staging&Team=QA" \
|
||
--data-binary @file.txt</code></pre>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Bucket Tagging</h3>
|
||
<pre class="mb-3"><code class="language-bash"># Set bucket tags
|
||
curl -X PUT "{{ api_base }}/<bucket>?tagging" \
|
||
-H "Content-Type: application/json" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||
-d '{
|
||
"TagSet": [
|
||
{"Key": "Environment", "Value": "Production"},
|
||
{"Key": "Team", "Value": "Engineering"}
|
||
]
|
||
}'</code></pre>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Use Cases</h3>
|
||
<div class="row g-2 mb-0">
|
||
<div class="col-md-6">
|
||
<ul class="small text-muted mb-0 ps-3">
|
||
<li>Filter objects for lifecycle expiration by tag</li>
|
||
<li>Use tag conditions in bucket policies</li>
|
||
</ul>
|
||
</div>
|
||
<div class="col-md-6">
|
||
<ul class="small text-muted mb-0 ps-3">
|
||
<li>Group objects by project or department</li>
|
||
<li>Trigger automation based on object tags</li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</article>
|
||
<article id="website-hosting" class="card shadow-sm docs-section">
|
||
<div class="card-body">
|
||
<div class="d-flex align-items-center gap-2 mb-3">
|
||
<span class="docs-section-kicker">27</span>
|
||
<h2 class="h4 mb-0">Static Website Hosting</h2>
|
||
</div>
|
||
<p class="text-muted">Host static websites directly from S3 buckets with custom index and error pages, served via custom domain mapping.</p>
|
||
|
||
<div class="alert alert-info small mb-3">
|
||
<strong>Prerequisite:</strong> Set <code>WEBSITE_HOSTING_ENABLED=true</code> to enable this feature.
|
||
</div>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">1. Configure bucket for website hosting</h3>
|
||
<pre class="mb-3"><code class="language-bash"># Enable website hosting with index and error documents
|
||
curl -X PUT "{{ api_base }}/<bucket>?website" \
|
||
-H "Content-Type: application/xml" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||
-d '<WebsiteConfiguration>
|
||
<IndexDocument><Suffix>index.html</Suffix></IndexDocument>
|
||
<ErrorDocument><Key>404.html</Key></ErrorDocument>
|
||
</WebsiteConfiguration>'
|
||
|
||
# Get website configuration
|
||
curl "{{ api_base }}/<bucket>?website" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"
|
||
|
||
# Remove website configuration
|
||
curl -X DELETE "{{ api_base }}/<bucket>?website" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"</code></pre>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">2. Map a custom domain to the bucket</h3>
|
||
<pre class="mb-3"><code class="language-bash"># Create domain mapping (admin only)
|
||
curl -X POST "{{ api_base }}/admin/website-domains" \
|
||
-H "Content-Type: application/json" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||
-d '{"domain": "example.com", "bucket": "my-site"}'
|
||
|
||
# List all domain mappings
|
||
curl "{{ api_base }}/admin/website-domains" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"
|
||
|
||
# Update a mapping
|
||
curl -X PUT "{{ api_base }}/admin/website-domains/example.com" \
|
||
-H "Content-Type: application/json" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||
-d '{"bucket": "new-site-bucket"}'
|
||
|
||
# Delete a mapping
|
||
curl -X DELETE "{{ api_base }}/admin/website-domains/example.com" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"</code></pre>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">3. Point your domain</h3>
|
||
<p class="small text-muted">MyFSIO handles domain routing natively via the <code>Host</code> header — no path-based proxy rules needed. Just point your domain to the MyFSIO API server.</p>
|
||
|
||
<div class="alert alert-secondary small mb-3">
|
||
<strong>Direct access (HTTP only):</strong> Point your domain's DNS (A or CNAME) directly to the MyFSIO server on port 5000.
|
||
</div>
|
||
|
||
<p class="small text-muted mb-2">For <strong>HTTPS</strong>, place a reverse proxy in front. The proxy only needs to forward traffic — MyFSIO handles the domain-to-bucket routing:</p>
|
||
<pre class="mb-3"><code class="language-nginx"># nginx example
|
||
server {
|
||
server_name example.com;
|
||
location / {
|
||
proxy_pass http://127.0.0.1:5000;
|
||
proxy_set_header Host $host; # Required: passes the domain to MyFSIO
|
||
proxy_set_header X-Real-IP $remote_addr;
|
||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||
proxy_set_header X-Forwarded-Proto $scheme;
|
||
}
|
||
}</code></pre>
|
||
<div class="alert alert-warning small mb-3">
|
||
<strong>Important:</strong> The <code>proxy_set_header Host $host;</code> directive is required. MyFSIO matches the incoming <code>Host</code> header against domain mappings to determine which bucket to serve.
|
||
</div>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">How it works</h3>
|
||
<div class="row g-2 mb-0">
|
||
<div class="col-md-6">
|
||
<ul class="small text-muted mb-0 ps-3">
|
||
<li><code>/</code> serves the configured index document</li>
|
||
<li><code>/about/</code> serves <code>about/index.html</code></li>
|
||
<li>Objects served with correct Content-Type</li>
|
||
</ul>
|
||
</div>
|
||
<div class="col-md-6">
|
||
<ul class="small text-muted mb-0 ps-3">
|
||
<li>Missing objects return the error document with 404</li>
|
||
<li>Website endpoints are public (no auth required)</li>
|
||
<li>Normal S3 API with auth continues to work</li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</article>
|
||
<article id="cors-config" class="card shadow-sm docs-section">
|
||
<div class="card-body">
|
||
<div class="d-flex align-items-center gap-2 mb-3">
|
||
<span class="docs-section-kicker">28</span>
|
||
<h2 class="h4 mb-0">CORS Configuration</h2>
|
||
</div>
|
||
<p class="text-muted">Configure per-bucket Cross-Origin Resource Sharing rules to control which origins can access your bucket from a browser.</p>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Setting CORS Rules</h3>
|
||
<pre class="mb-3"><code class="language-bash"># Set CORS configuration
|
||
curl -X PUT "{{ api_base }}/<bucket>?cors" \
|
||
-H "Content-Type: application/xml" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>" \
|
||
-d '<CORSConfiguration>
|
||
<CORSRule>
|
||
<AllowedOrigin>https://example.com</AllowedOrigin>
|
||
<AllowedMethod>GET</AllowedMethod>
|
||
<AllowedMethod>PUT</AllowedMethod>
|
||
<AllowedHeader>*</AllowedHeader>
|
||
<ExposeHeader>ETag</ExposeHeader>
|
||
<MaxAgeSeconds>3600</MaxAgeSeconds>
|
||
</CORSRule>
|
||
</CORSConfiguration>'
|
||
|
||
# Get CORS configuration
|
||
curl "{{ api_base }}/<bucket>?cors" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"
|
||
|
||
# Delete CORS configuration
|
||
curl -X DELETE "{{ api_base }}/<bucket>?cors" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"</code></pre>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Rule Fields</h3>
|
||
<div class="table-responsive mb-0">
|
||
<table class="table table-sm table-bordered small mb-0">
|
||
<thead class="table-light">
|
||
<tr>
|
||
<th>Field</th>
|
||
<th>Description</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr>
|
||
<td><code>AllowedOrigin</code></td>
|
||
<td>Origins allowed to make requests (supports <code>*</code> wildcard)</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>AllowedMethod</code></td>
|
||
<td>HTTP methods: <code>GET</code>, <code>PUT</code>, <code>POST</code>, <code>DELETE</code>, <code>HEAD</code></td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>AllowedHeader</code></td>
|
||
<td>Request headers allowed in preflight (supports <code>*</code>)</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>ExposeHeader</code></td>
|
||
<td>Response headers visible to the browser (e.g., <code>ETag</code>, <code>x-amz-request-id</code>)</td>
|
||
</tr>
|
||
<tr>
|
||
<td><code>MaxAgeSeconds</code></td>
|
||
<td>How long the browser caches preflight results</td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
</div>
|
||
</div>
|
||
</article>
|
||
<article id="post-object" class="card shadow-sm docs-section">
|
||
<div class="card-body">
|
||
<div class="d-flex align-items-center gap-2 mb-3">
|
||
<span class="docs-section-kicker">29</span>
|
||
<h2 class="h4 mb-0">PostObject (HTML Form Upload)</h2>
|
||
</div>
|
||
<p class="text-muted">Upload objects directly from an HTML form using browser-based POST uploads with policy-based authorization.</p>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Form Fields</h3>
|
||
<div class="table-responsive mb-3">
|
||
<table class="table table-sm table-bordered small">
|
||
<thead class="table-light">
|
||
<tr>
|
||
<th>Field</th>
|
||
<th>Description</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr><td><code>key</code></td><td>Object key (supports <code>${filename}</code> variable)</td></tr>
|
||
<tr><td><code>file</code></td><td>The file to upload</td></tr>
|
||
<tr><td><code>policy</code></td><td>Base64-encoded policy document (JSON)</td></tr>
|
||
<tr><td><code>x-amz-signature</code></td><td>HMAC-SHA256 signature of the policy</td></tr>
|
||
<tr><td><code>x-amz-credential</code></td><td>Access key / date / region / s3 / aws4_request</td></tr>
|
||
<tr><td><code>x-amz-algorithm</code></td><td><code>AWS4-HMAC-SHA256</code></td></tr>
|
||
<tr><td><code>x-amz-date</code></td><td>ISO 8601 date (e.g., <code>20250101T000000Z</code>)</td></tr>
|
||
<tr><td><code>Content-Type</code></td><td>MIME type of the uploaded file</td></tr>
|
||
<tr><td><code>x-amz-meta-*</code></td><td>Custom metadata headers</td></tr>
|
||
</tbody>
|
||
</table>
|
||
</div>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Simple Upload (No Signing)</h3>
|
||
<pre class="mb-3"><code class="language-html"><form action="{{ api_base }}/my-bucket" method="POST" enctype="multipart/form-data">
|
||
<input type="hidden" name="key" value="uploads/${filename}">
|
||
<input type="file" name="file">
|
||
<button type="submit">Upload</button>
|
||
</form></code></pre>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Signed Upload (With Policy)</h3>
|
||
<p class="small text-muted mb-0">For authenticated uploads, include a base64-encoded policy and SigV4 signature fields. The policy constrains allowed keys, content types, and size limits. See docs.md Section 20 for full signing examples.</p>
|
||
</div>
|
||
</article>
|
||
<article id="list-objects-v2" class="card shadow-sm docs-section">
|
||
<div class="card-body">
|
||
<div class="d-flex align-items-center gap-2 mb-3">
|
||
<span class="docs-section-kicker">30</span>
|
||
<h2 class="h4 mb-0">List Objects API v2</h2>
|
||
</div>
|
||
<p class="text-muted">Use the v2 list API for improved pagination with continuation tokens instead of markers.</p>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Usage</h3>
|
||
<pre class="mb-3"><code class="language-bash"># List with v2 API
|
||
curl "{{ api_base }}/<bucket>?list-type=2&prefix=logs/&delimiter=/&max-keys=100" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"
|
||
|
||
# Paginate with continuation token
|
||
curl "{{ api_base }}/<bucket>?list-type=2&continuation-token=<token>" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"
|
||
|
||
# Start listing after a specific key
|
||
curl "{{ api_base }}/<bucket>?list-type=2&start-after=photos/2025/" \
|
||
-H "X-Access-Key: <key>" -H "X-Secret-Key: <secret>"</code></pre>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Query Parameters</h3>
|
||
<div class="table-responsive mb-0">
|
||
<table class="table table-sm table-bordered small mb-0">
|
||
<thead class="table-light">
|
||
<tr>
|
||
<th>Parameter</th>
|
||
<th>Description</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr><td><code>list-type=2</code></td><td>Enables v2 API (required)</td></tr>
|
||
<tr><td><code>prefix</code></td><td>Filter to keys starting with this prefix</td></tr>
|
||
<tr><td><code>delimiter</code></td><td>Group keys by delimiter (typically <code>/</code> for folders)</td></tr>
|
||
<tr><td><code>max-keys</code></td><td>Maximum objects to return (default 1000)</td></tr>
|
||
<tr><td><code>continuation-token</code></td><td>Token from previous response for pagination</td></tr>
|
||
<tr><td><code>start-after</code></td><td>Start listing after this key (first page only)</td></tr>
|
||
<tr><td><code>fetch-owner</code></td><td>Include owner info in response</td></tr>
|
||
<tr><td><code>encoding-type</code></td><td>Set to <code>url</code> to URL-encode keys in response</td></tr>
|
||
</tbody>
|
||
</table>
|
||
</div>
|
||
</div>
|
||
</article>
|
||
<article id="upgrading" class="card shadow-sm docs-section">
|
||
<div class="card-body">
|
||
<div class="d-flex align-items-center gap-2 mb-3">
|
||
<span class="docs-section-kicker">31</span>
|
||
<h2 class="h4 mb-0">Upgrading & Updates</h2>
|
||
</div>
|
||
<p class="text-muted">How to safely update MyFSIO to a new version.</p>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Pre-Update Backup</h3>
|
||
<p class="small text-muted">Always back up before updating:</p>
|
||
<pre class="mb-3"><code class="language-bash"># Back up configuration
|
||
cp -r data/.myfsio.sys/config/ config-backup/
|
||
|
||
# Back up data (optional, for critical deployments)
|
||
tar czf myfsio-backup-$(date +%Y%m%d).tar.gz data/
|
||
|
||
# Back up logs
|
||
cp -r logs/ logs-backup/</code></pre>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Update Procedure</h3>
|
||
<ol class="docs-steps mb-3">
|
||
<li><strong>Stop the service:</strong> <code>sudo systemctl stop myfsio</code> (or kill the process)</li>
|
||
<li><strong>Pull new version:</strong> <code>git pull origin main</code> or download the new binary</li>
|
||
<li><strong>Install dependencies:</strong> <code>pip install -r requirements.txt</code></li>
|
||
<li><strong>Validate config:</strong> <code>python run.py --check-config</code></li>
|
||
<li><strong>Start the service:</strong> <code>sudo systemctl start myfsio</code></li>
|
||
<li><strong>Verify:</strong> <code>curl http://localhost:5000/myfsio/health</code></li>
|
||
</ol>
|
||
|
||
<h3 class="h6 text-uppercase text-muted mt-4">Rollback</h3>
|
||
<p class="small text-muted mb-0">If something goes wrong, stop the service, restore the backed-up config and data directories, then restart with the previous binary or code version. See <code>docs.md</code> Section 4 for detailed rollback procedures including blue-green deployment strategies.</p>
|
||
</div>
|
||
</article>
|
||
<article id="api-matrix" class="card shadow-sm docs-section">
|
||
<div class="card-body">
|
||
<div class="d-flex align-items-center gap-2 mb-3">
|
||
<span class="docs-section-kicker">32</span>
|
||
<h2 class="h4 mb-0">Full API Reference</h2>
|
||
</div>
|
||
<p class="text-muted">Complete list of all S3-compatible, admin, and KMS endpoints.</p>
|
||
<pre class="mb-0"><code class="language-text"># Service
|
||
GET /myfsio/health # Health check
|
||
|
||
# Bucket Operations
|
||
GET / # List buckets
|
||
PUT /<bucket> # Create bucket
|
||
DELETE /<bucket> # Delete bucket
|
||
GET /<bucket> # List objects (?list-type=2)
|
||
HEAD /<bucket> # Check bucket exists
|
||
POST /<bucket> # POST object / form upload
|
||
POST /<bucket>?delete # Bulk delete
|
||
|
||
# Bucket Configuration
|
||
GET|PUT|DELETE /<bucket>?policy # Bucket policy
|
||
GET|PUT /<bucket>?quota # Bucket quota
|
||
GET|PUT /<bucket>?versioning # Versioning
|
||
GET|PUT|DELETE /<bucket>?lifecycle # Lifecycle rules
|
||
GET|PUT|DELETE /<bucket>?cors # CORS config
|
||
GET|PUT|DELETE /<bucket>?encryption # Default encryption
|
||
GET|PUT /<bucket>?acl # Bucket ACL
|
||
GET|PUT|DELETE /<bucket>?tagging # Bucket tags
|
||
GET|PUT|DELETE /<bucket>?replication # Replication rules
|
||
GET|PUT /<bucket>?logging # Access logging
|
||
GET|PUT /<bucket>?notification # Event notifications
|
||
GET|PUT /<bucket>?object-lock # Object lock config
|
||
GET|PUT|DELETE /<bucket>?website # Static website
|
||
GET /<bucket>?uploads # List multipart uploads
|
||
GET /<bucket>?versions # List object versions
|
||
GET /<bucket>?location # Bucket region
|
||
|
||
# Object Operations
|
||
PUT /<bucket>/<key> # Upload object
|
||
GET /<bucket>/<key> # Download (Range supported)
|
||
DELETE /<bucket>/<key> # Delete object
|
||
HEAD /<bucket>/<key> # Object metadata
|
||
POST /<bucket>/<key>?select # SQL query (SelectObjectContent)
|
||
|
||
# Object Configuration
|
||
GET|PUT|DELETE /<bucket>/<key>?tagging # Object tags
|
||
GET|PUT /<bucket>/<key>?acl # Object ACL
|
||
GET|PUT /<bucket>/<key>?retention # Object retention
|
||
GET|PUT /<bucket>/<key>?legal-hold # Legal hold
|
||
|
||
# Multipart Upload
|
||
POST /<bucket>/<key>?uploads # Initiate
|
||
PUT /<bucket>/<key>?uploadId=X&partNumber=N # Upload part
|
||
POST /<bucket>/<key>?uploadId=X # Complete
|
||
DELETE /<bucket>/<key>?uploadId=X # Abort
|
||
GET /<bucket>/<key>?uploadId=X # List parts
|
||
|
||
# Copy (via x-amz-copy-source header)
|
||
PUT /<bucket>/<key> # CopyObject
|
||
PUT /<bucket>/<key>?uploadId&partNumber # UploadPartCopy
|
||
|
||
# Admin API
|
||
GET|PUT /admin/site # Local site config
|
||
GET /admin/sites # List peers
|
||
POST /admin/sites # Register peer
|
||
GET|PUT|DELETE /admin/sites/<id> # Manage peer
|
||
GET /admin/sites/<id>/health # Peer health
|
||
GET /admin/topology # Cluster topology
|
||
GET|POST|PUT|DELETE /admin/website-domains # Domain mappings
|
||
|
||
# KMS API
|
||
GET|POST /kms/keys # List / Create keys
|
||
GET|DELETE /kms/keys/<id> # Get / Delete key
|
||
POST /kms/keys/<id>/enable # Enable key
|
||
POST /kms/keys/<id>/disable # Disable key
|
||
POST /kms/keys/<id>/rotate # Rotate key
|
||
POST /kms/encrypt # Encrypt data
|
||
POST /kms/decrypt # Decrypt data
|
||
POST /kms/generate-data-key # Generate data key
|
||
POST /kms/generate-random # Generate random bytes</code></pre>
|
||
</div>
|
||
</article>
|
||
</div>
|
||
<div class="col-xl-4 docs-sidebar-col">
|
||
<aside class="card shadow-sm docs-sidebar">
|
||
<div class="card-body">
|
||
<h3 class="h6 text-uppercase text-muted mb-3">On this page</h3>
|
||
<ul class="list-unstyled docs-toc mb-4">
|
||
<li><a href="#setup">Set up & run</a></li>
|
||
<li><a href="#background">Running in background</a></li>
|
||
<li><a href="#auth">Authentication & IAM</a></li>
|
||
<li><a href="#console">Console tour</a></li>
|
||
<li><a href="#automation">Automation / CLI</a></li>
|
||
<li><a href="#api">REST endpoints</a></li>
|
||
<li><a href="#examples">API Examples</a></li>
|
||
<li><a href="#replication">Site Replication & Sync</a></li>
|
||
<li><a href="#site-registry">Site Registry</a></li>
|
||
<li><a href="#versioning">Object Versioning</a></li>
|
||
<li><a href="#quotas">Bucket Quotas</a></li>
|
||
<li><a href="#encryption">Encryption</a></li>
|
||
<li><a href="#lifecycle">Lifecycle Rules</a></li>
|
||
<li><a href="#garbage-collection">Garbage Collection</a></li>
|
||
<li><a href="#metrics">Metrics History</a></li>
|
||
<li><a href="#operation-metrics">Operation Metrics</a></li>
|
||
<li><a href="#troubleshooting">Troubleshooting</a></li>
|
||
<li><a href="#health-check">Health Check</a></li>
|
||
<li><a href="#object-lock">Object Lock & Retention</a></li>
|
||
<li><a href="#access-logging">Access Logging</a></li>
|
||
<li><a href="#notifications">Notifications & Webhooks</a></li>
|
||
<li><a href="#select-content">SelectObjectContent</a></li>
|
||
<li><a href="#advanced-ops">Advanced Operations</a></li>
|
||
<li><a href="#acls">Access Control Lists</a></li>
|
||
<li><a href="#tagging">Object & Bucket Tagging</a></li>
|
||
<li><a href="#website-hosting">Static Website Hosting</a></li>
|
||
<li><a href="#cors-config">CORS Configuration</a></li>
|
||
<li><a href="#post-object">PostObject (Form Upload)</a></li>
|
||
<li><a href="#list-objects-v2">List Objects API v2</a></li>
|
||
<li><a href="#upgrading">Upgrading & Updates</a></li>
|
||
<li><a href="#api-matrix">Full API Reference</a></li>
|
||
</ul>
|
||
<div class="docs-sidebar-callouts">
|
||
<div>
|
||
<div class="small text-uppercase text-muted">API base</div>
|
||
<code class="d-block">{{ api_base }}</code>
|
||
</div>
|
||
<div>
|
||
<div class="small text-uppercase text-muted">Initial credentials</div>
|
||
<span class="text-muted small">Generated on first run (check console)</span>
|
||
</div>
|
||
<div>
|
||
<div class="small text-uppercase text-muted">Logs</div>
|
||
<span class="text-muted small">logs/api.log · logs/ui.log</span>
|
||
</div>
|
||
</div>
|
||
<div class="mt-4">
|
||
<p class="small text-muted mb-1">Need more automation? Extend <code>app/s3_api.py</code> or wrap <code>run_api.py</code> with gunicorn for production-style deployments.</p>
|
||
</div>
|
||
</div>
|
||
</aside>
|
||
</div>
|
||
</div>
|
||
{% endblock %}
|