Debug replication corruption issue

app/replication.py

@@ -2,6 +2,7 @@
 from __future__ import annotations
 
 import logging
+import mimetypes
 import threading
 from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass
@@ -10,9 +11,10 @@ from typing import Dict, Optional
 
 import boto3
 from botocore.exceptions import ClientError
+from boto3.exceptions import S3UploadFailedError
 
 from .connections import ConnectionStore, RemoteConnection
-from .storage import ObjectStorage
+from .storage import ObjectStorage, StorageError
 
 logger = logging.getLogger(__name__)
 
@@ -116,21 +118,73 @@ class ReplicationManager:
             # We need the file content. 
             # Since ObjectStorage is filesystem based, let's get the stream.
             # We need to be careful about closing it.
-            meta = self.storage.get_object_meta(bucket_name, object_key)
+            try:
-            if not meta:
+                path = self.storage.get_object_path(bucket_name, object_key)
+            except StorageError:
                 return
 
-            with self.storage.open_object(bucket_name, object_key) as f:
+            metadata = self.storage.get_object_metadata(bucket_name, object_key)
-                extra_args = {}
+
-                if meta.metadata:
+            extra_args = {}
-                    extra_args["Metadata"] = meta.metadata
+            if metadata:
-                
+                extra_args["Metadata"] = metadata
-                s3.upload_fileobj(
+            
-                    f,
+            # Guess content type to prevent corruption/wrong handling
-                    rule.target_bucket,
+            content_type, _ = mimetypes.guess_type(path)
-                    object_key,
+            file_size = path.stat().st_size
-                    ExtraArgs=extra_args
+
-                )
+            # Debug: Calculate MD5 of source file
+            import hashlib
+            md5_hash = hashlib.md5()
+            with path.open("rb") as f:
+                # Log first 32 bytes
+                header = f.read(32)
+                logger.info(f"Source first 32 bytes: {header.hex()}")
+                md5_hash.update(header)
+                for chunk in iter(lambda: f.read(4096), b""):
+                    md5_hash.update(chunk)
+            source_md5 = md5_hash.hexdigest()
+            logger.info(f"Replicating {bucket_name}/{object_key}: Size={file_size}, MD5={source_md5}, ContentType={content_type}")
+
+            try:
+                with path.open("rb") as f:
+                    s3.put_object(
+                        Bucket=rule.target_bucket,
+                        Key=object_key,
+                        Body=f,
+                        ContentLength=file_size,
+                        ContentType=content_type or "application/octet-stream",
+                        Metadata=metadata or {}
+                    )
+            except (ClientError, S3UploadFailedError) as e:
+                # Check if it's a NoSuchBucket error (either direct or wrapped)
+                is_no_bucket = False
+                if isinstance(e, ClientError):
+                    if e.response['Error']['Code'] == 'NoSuchBucket':
+                        is_no_bucket = True
+                elif isinstance(e, S3UploadFailedError):
+                    if "NoSuchBucket" in str(e):
+                        is_no_bucket = True
+
+                if is_no_bucket:
+                    logger.info(f"Target bucket {rule.target_bucket} not found. Attempting to create it.")
+                    try:
+                        s3.create_bucket(Bucket=rule.target_bucket)
+                        # Retry upload
+                        with path.open("rb") as f:
+                            s3.put_object(
+                                Bucket=rule.target_bucket,
+                                Key=object_key,
+                                Body=f,
+                                ContentLength=file_size,
+                                ContentType=content_type or "application/octet-stream",
+                                Metadata=metadata or {}
+                            )
+                    except Exception as create_err:
+                        logger.error(f"Failed to create target bucket {rule.target_bucket}: {create_err}")
+                        raise e # Raise original error
+                else:
+                    raise e
             
             logger.info(f"Replicated {bucket_name}/{object_key} to {conn.name} ({rule.target_bucket})")
 

app/s3_api.py

@@ -1078,7 +1078,13 @@ def object_handler(bucket_name: str, object_key: str):
         _, error = _object_principal("write", bucket_name, object_key)
         if error:
             return error
-        stream = request.stream
+        
+        # Debug: Log incoming request details
+        current_app.logger.info(f"Receiving PUT {bucket_name}/{object_key}")
+        current_app.logger.info(f"Headers: {dict(request.headers)}")
+        current_app.logger.info(f"Content-Length: {request.content_length}")
+        
+        stream = DebugStream(request.stream, current_app.logger)
         metadata = _extract_request_metadata()
         try:
             meta = storage.put_object(
@@ -1252,3 +1258,19 @@ def head_object(bucket_name: str, object_key: str) -> Response:
         return _error_response("NoSuchKey", "Object not found", 404)
     except IamError as exc:
         return _error_response("AccessDenied", str(exc), 403)
+
+
+class DebugStream:
+    def __init__(self, stream, logger):
+        self.stream = stream
+        self.logger = logger
+        self.first_chunk = True
+
+    def read(self, size=-1):
+        chunk = self.stream.read(size)
+        if self.first_chunk and chunk:
+            # Log first 32 bytes
+            prefix = chunk[:32]
+            self.logger.info(f"Received first 32 bytes: {prefix.hex()}")
+            self.first_chunk = False
+        return chunk

app/ui.py

@@ -6,7 +6,9 @@ import uuid
 from typing import Any
 from urllib.parse import urlparse
 
+import boto3
 import requests
+from botocore.exceptions import ClientError
 from flask import (
     Blueprint,
     Response,
@@ -1070,6 +1072,73 @@ def create_connection():
     return redirect(url_for("ui.connections_dashboard"))
 
 
+@ui_bp.post("/connections/test")
+def test_connection():
+    principal = _current_principal()
+    try:
+        _iam().authorize(principal, None, "iam:list_users")
+    except IamError:
+        return jsonify({"status": "error", "message": "Access denied"}), 403
+
+    data = request.get_json(silent=True) or request.form
+    endpoint = data.get("endpoint_url", "").strip()
+    access_key = data.get("access_key", "").strip()
+    secret_key = data.get("secret_key", "").strip()
+    region = data.get("region", "us-east-1").strip()
+
+    if not all([endpoint, access_key, secret_key]):
+        return jsonify({"status": "error", "message": "Missing credentials"}), 400
+
+    try:
+        s3 = boto3.client(
+            "s3",
+            endpoint_url=endpoint,
+            aws_access_key_id=access_key,
+            aws_secret_access_key=secret_key,
+            region_name=region,
+        )
+        # Try to list buckets to verify credentials and endpoint
+        s3.list_buckets()
+        return jsonify({"status": "ok", "message": "Connection successful"})
+    except Exception as e:
+        return jsonify({"status": "error", "message": str(e)}), 400
+
+
+@ui_bp.post("/connections/<connection_id>/update")
+def update_connection(connection_id: str):
+    principal = _current_principal()
+    try:
+        _iam().authorize(principal, None, "iam:list_users")
+    except IamError:
+        flash("Access denied", "danger")
+        return redirect(url_for("ui.buckets_overview"))
+
+    conn = _connections().get(connection_id)
+    if not conn:
+        flash("Connection not found", "danger")
+        return redirect(url_for("ui.connections_dashboard"))
+
+    name = request.form.get("name", "").strip()
+    endpoint = request.form.get("endpoint_url", "").strip()
+    access_key = request.form.get("access_key", "").strip()
+    secret_key = request.form.get("secret_key", "").strip()
+    region = request.form.get("region", "us-east-1").strip()
+
+    if not all([name, endpoint, access_key, secret_key]):
+        flash("All fields are required", "danger")
+        return redirect(url_for("ui.connections_dashboard"))
+
+    conn.name = name
+    conn.endpoint_url = endpoint
+    conn.access_key = access_key
+    conn.secret_key = secret_key
+    conn.region = region
+    
+    _connections().save()
+    flash(f"Connection '{name}' updated", "success")
+    return redirect(url_for("ui.connections_dashboard"))
+
+
 @ui_bp.post("/connections/<connection_id>/delete")
 def delete_connection(connection_id: str):
     principal = _current_principal()
@@ -1105,16 +1174,6 @@ def update_bucket_replication(bucket_name: str):
         if not target_conn_id or not target_bucket:
             flash("Target connection and bucket are required", "danger")
         else:
-            # Check if user wants to create the remote bucket
-            create_remote = request.form.get("create_remote_bucket") == "on"
-            if create_remote:
-                try:
-                    _replication().create_remote_bucket(target_conn_id, target_bucket)
-                    flash(f"Created remote bucket '{target_bucket}'", "success")
-                except Exception as e:
-                    flash(f"Failed to create remote bucket: {e}", "warning")
-                    # We continue to set the rule even if creation fails (maybe it exists?)
-
             rule = ReplicationRule(
                 bucket_name=bucket_name,
                 target_connection_id=target_conn_id,

templates/bucket_detail.html

@@ -408,11 +408,9 @@
                 </dl>
               </div>
               
-              <form method="POST" action="{{ url_for('ui.update_bucket_replication', bucket_name=bucket_name) }}" onsubmit="return confirm('Are you sure you want to disable replication?');">
+              <button type="button" class="btn btn-danger" data-bs-toggle="modal" data-bs-target="#disableReplicationModal">
-                <input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
+                Disable Replication
-                <input type="hidden" name="action" value="delete">
+              </button>
-                <button type="submit" class="btn btn-danger">Disable Replication</button>
-              </form>
               
               {% else %}
               <p class="text-muted">Replication allows you to automatically copy new objects from this bucket to a bucket in another S3-compatible service.</p>
@@ -436,12 +434,7 @@
                 <div class="mb-3">
                   <label for="target_bucket" class="form-label">Target Bucket Name</label>
                   <input type="text" class="form-control" id="target_bucket" name="target_bucket" required placeholder="e.g. my-backup-bucket">
-                  <div class="form-check mt-2">
+                  <div class="form-text">If the target bucket does not exist, it will be created automatically.</div>
-                    <input class="form-check-input" type="checkbox" id="create_remote_bucket" name="create_remote_bucket">
-                    <label class="form-check-label" for="create_remote_bucket">
-                      Create this bucket on the remote server if it doesn't exist
-                    </label>
-                  </div>
                 </div>
                 
                 <button type="submit" class="btn btn-primary">Enable Replication</button>
@@ -715,6 +708,30 @@
     </div>
   </div>
 </div>
+
+<!-- Disable Replication Modal -->
+<div class="modal fade" id="disableReplicationModal" tabindex="-1" aria-hidden="true">
+  <div class="modal-dialog modal-dialog-centered">
+    <div class="modal-content">
+      <div class="modal-header">
+        <h5 class="modal-title">Disable Replication</h5>
+        <button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
+      </div>
+      <div class="modal-body">
+        <p>Are you sure you want to disable replication for this bucket?</p>
+        <p class="text-muted small">Existing objects in the target bucket will remain, but new uploads will no longer be copied.</p>
+      </div>
+      <div class="modal-footer">
+        <button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Cancel</button>
+        <form method="POST" action="{{ url_for('ui.update_bucket_replication', bucket_name=bucket_name) }}">
+            <input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
+            <input type="hidden" name="action" value="delete">
+            <button type="submit" class="btn btn-danger">Disable Replication</button>
+        </form>
+      </div>
+    </div>
+  </div>
+</div>
 {% endblock %}
 
 {% block extra_scripts %}

templates/connections.html

@@ -12,12 +12,12 @@
 
 <div class="row">
     <div class="col-md-4">
-        <div class="card">
+        <div class="card shadow-sm">
-            <div class="card-header">
+            <div class="card-header fw-semibold">
                 Add New Connection
             </div>
             <div class="card-body">
-                <form method="POST" action="{{ url_for('ui.create_connection') }}">
+                <form method="POST" action="{{ url_for('ui.create_connection') }}" id="createConnectionForm">
                     <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
                     <div class="mb-3">
                         <label for="name" class="form-label">Name</label>
@@ -37,44 +37,69 @@
                     </div>
                     <div class="mb-3">
                         <label for="secret_key" class="form-label">Secret Key</label>
-                        <input type="password" class="form-control" id="secret_key" name="secret_key" required>
+                        <div class="input-group">
+                            <input type="password" class="form-control" id="secret_key" name="secret_key" required>
+                            <button class="btn btn-outline-secondary" type="button" onclick="togglePassword('secret_key')">
+                                <i class="bi bi-eye"></i> Show
+                            </button>
+                        </div>
                     </div>
-                    <button type="submit" class="btn btn-primary">Add Connection</button>
+                    <div class="d-grid gap-2">
+                        <button type="button" class="btn btn-outline-secondary" id="testConnectionBtn">Test Connection</button>
+                        <button type="submit" class="btn btn-primary">Add Connection</button>
+                    </div>
+                    <div id="testResult" class="mt-2"></div>
                 </form>
             </div>
         </div>
     </div>
     
     <div class="col-md-8">
-        <div class="card">
+        <div class="card shadow-sm">
-            <div class="card-header">
+            <div class="card-header fw-semibold">
                 Existing Connections
             </div>
             <div class="card-body">
                 {% if connections %}
                 <div class="table-responsive">
-                    <table class="table table-hover">
+                    <table class="table table-hover align-middle">
                         <thead>
                             <tr>
                                 <th>Name</th>
                                 <th>Endpoint</th>
                                 <th>Region</th>
                                 <th>Access Key</th>
-                                <th>Actions</th>
+                                <th class="text-end">Actions</th>
                             </tr>
                         </thead>
                         <tbody>
                             {% for conn in connections %}
                             <tr>
-                                <td>{{ conn.name }}</td>
+                                <td class="fw-medium">{{ conn.name }}</td>
-                                <td>{{ conn.endpoint_url }}</td>
+                                <td class="small text-muted">{{ conn.endpoint_url }}</td>
-                                <td>{{ conn.region }}</td>
+                                <td><span class="badge bg-light text-dark border">{{ conn.region }}</span></td>
-                                <td><code>{{ conn.access_key }}</code></td>
+                                <td><code class="small">{{ conn.access_key }}</code></td>
-                                <td>
+                                <td class="text-end">
-                                    <form method="POST" action="{{ url_for('ui.delete_connection', connection_id=conn.id) }}" onsubmit="return confirm('Are you sure?');" style="display: inline;">
+                                    <div class="btn-group">
-                                        <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
+                                        <button type="button" class="btn btn-sm btn-outline-primary" 
-                                        <button type="submit" class="btn btn-sm btn-danger">Delete</button>
+                                            data-bs-toggle="modal" 
-                                    </form>
+                                            data-bs-target="#editConnectionModal"
+                                            data-id="{{ conn.id }}"
+                                            data-name="{{ conn.name }}"
+                                            data-endpoint="{{ conn.endpoint_url }}"
+                                            data-region="{{ conn.region }}"
+                                            data-access="{{ conn.access_key }}"
+                                            data-secret="{{ conn.secret_key }}">
+                                            Edit
+                                        </button>
+                                        <button type="button" class="btn btn-sm btn-outline-danger" 
+                                            data-bs-toggle="modal" 
+                                            data-bs-target="#deleteConnectionModal"
+                                            data-id="{{ conn.id }}"
+                                            data-name="{{ conn.name }}">
+                                            Delete
+                                        </button>
+                                    </div>
                                 </td>
                             </tr>
                             {% endfor %}
@@ -82,10 +107,164 @@
                     </table>
                 </div>
                 {% else %}
-                <p class="text-muted text-center my-4">No remote connections configured.</p>
+                <div class="text-center py-5 text-muted">
+                    <svg xmlns="http://www.w3.org/2000/svg" width="48" height="48" fill="currentColor" class="bi bi-hdd-network mb-3" viewBox="0 0 16 16">
+                        <path d="M4.5 5a.5.5 0 1 0 0-1 .5.5 0 0 0 0 1zM3 4.5a.5.5 0 1 1-1 0 .5.5 0 0 1 1 0z"/>
+                        <path d="M0 4a2 2 0 0 1 2-2h12a2 2 0 0 1 2 2v1a2 2 0 0 1-2 2H8.5v3a1.5 1.5 0 0 1 1.5 1.5v3.375a1.125 1.125 0 0 1-1.125 1.125h-1.75a1.125 1.125 0 0 1-1.125-1.125V11.5A1.5 1.5 0 0 1 7.5 10V7H2a2 2 0 0 1-2-2V4zm1 0v1a1 1 0 0 0 1 1h12a1 1 0 0 0 1-1V4a1 1 0 0 0-1-1H2a1 1 0 0 0-1 1zm6 5v1.5a.5.5 0 0 0 .5.5h1.75a.5.5 0 0 0 .5-.5V10a.5.5 0 0 0-.5-.5H7.5a.5.5 0 0 0-.5.5z"/>
+                    </svg>
+                    <p>No remote connections configured.</p>
+                </div>
                 {% endif %}
             </div>
         </div>
     </div>
 </div>
+
+<!-- Edit Connection Modal -->
+<div class="modal fade" id="editConnectionModal" tabindex="-1" aria-hidden="true">
+    <div class="modal-dialog">
+        <div class="modal-content">
+            <div class="modal-header">
+                <h5 class="modal-title">Edit Connection</h5>
+                <button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
+            </div>
+            <form method="POST" id="editConnectionForm">
+                <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
+                <div class="modal-body">
+                    <div class="mb-3">
+                        <label for="edit_name" class="form-label">Name</label>
+                        <input type="text" class="form-control" id="edit_name" name="name" required>
+                    </div>
+                    <div class="mb-3">
+                        <label for="edit_endpoint_url" class="form-label">Endpoint URL</label>
+                        <input type="url" class="form-control" id="edit_endpoint_url" name="endpoint_url" required>
+                    </div>
+                    <div class="mb-3">
+                        <label for="edit_region" class="form-label">Region</label>
+                        <input type="text" class="form-control" id="edit_region" name="region" required>
+                    </div>
+                    <div class="mb-3">
+                        <label for="edit_access_key" class="form-label">Access Key</label>
+                        <input type="text" class="form-control" id="edit_access_key" name="access_key" required>
+                    </div>
+                    <div class="mb-3">
+                        <label for="edit_secret_key" class="form-label">Secret Key</label>
+                        <div class="input-group">
+                            <input type="password" class="form-control" id="edit_secret_key" name="secret_key" required>
+                            <button class="btn btn-outline-secondary" type="button" onclick="togglePassword('edit_secret_key')">
+                                Show
+                            </button>
+                        </div>
+                    </div>
+                    <div id="editTestResult" class="mt-2"></div>
+                </div>
+                <div class="modal-footer">
+                    <button type="button" class="btn btn-outline-secondary" id="editTestConnectionBtn">Test Connection</button>
+                    <button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Cancel</button>
+                    <button type="submit" class="btn btn-primary">Save Changes</button>
+                </div>
+            </form>
+        </div>
+    </div>
+</div>
+
+<!-- Delete Connection Modal -->
+<div class="modal fade" id="deleteConnectionModal" tabindex="-1" aria-hidden="true">
+    <div class="modal-dialog modal-dialog-centered">
+        <div class="modal-content">
+            <div class="modal-header">
+                <h5 class="modal-title">Delete Connection</h5>
+                <button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
+            </div>
+            <div class="modal-body">
+                <p>Are you sure you want to delete the connection <strong id="deleteConnectionName"></strong>?</p>
+                <p class="text-muted small">This will stop any replication rules using this connection.</p>
+            </div>
+            <div class="modal-footer">
+                <button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Cancel</button>
+                <form method="POST" id="deleteConnectionForm">
+                    <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
+                    <button type="submit" class="btn btn-danger">Delete</button>
+                </form>
+            </div>
+        </div>
+    </div>
+</div>
+
+<script>
+    function togglePassword(id) {
+        const input = document.getElementById(id);
+        if (input.type === "password") {
+            input.type = "text";
+        } else {
+            input.type = "password";
+        }
+    }
+
+    // Test Connection Logic
+    async function testConnection(formId, resultId) {
+        const form = document.getElementById(formId);
+        const resultDiv = document.getElementById(resultId);
+        const formData = new FormData(form);
+        const data = Object.fromEntries(formData.entries());
+        
+        resultDiv.innerHTML = '<div class="text-info"><span class="spinner-border spinner-border-sm" role="status" aria-hidden="true"></span> Testing...</div>';
+        
+        try {
+            const response = await fetch("{{ url_for('ui.test_connection') }}", {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                    "X-CSRFToken": "{{ csrf_token() }}"
+                },
+                body: JSON.stringify(data)
+            });
+            
+            const result = await response.json();
+            if (response.ok) {
+                resultDiv.innerHTML = `<div class="text-success"><i class="bi bi-check-circle"></i> ${result.message}</div>`;
+            } else {
+                resultDiv.innerHTML = `<div class="text-danger"><i class="bi bi-exclamation-circle"></i> ${result.message}</div>`;
+            }
+        } catch (error) {
+            resultDiv.innerHTML = `<div class="text-danger"><i class="bi bi-exclamation-circle"></i> Connection failed</div>`;
+        }
+    }
+
+    document.getElementById('testConnectionBtn').addEventListener('click', () => {
+        testConnection('createConnectionForm', 'testResult');
+    });
+
+    document.getElementById('editTestConnectionBtn').addEventListener('click', () => {
+        testConnection('editConnectionForm', 'editTestResult');
+    });
+
+    // Modal Event Listeners
+    const editModal = document.getElementById('editConnectionModal');
+    editModal.addEventListener('show.bs.modal', event => {
+        const button = event.relatedTarget;
+        const id = button.getAttribute('data-id');
+        
+        document.getElementById('edit_name').value = button.getAttribute('data-name');
+        document.getElementById('edit_endpoint_url').value = button.getAttribute('data-endpoint');
+        document.getElementById('edit_region').value = button.getAttribute('data-region');
+        document.getElementById('edit_access_key').value = button.getAttribute('data-access');
+        document.getElementById('edit_secret_key').value = button.getAttribute('data-secret');
+        document.getElementById('editTestResult').innerHTML = '';
+        
+        const form = document.getElementById('editConnectionForm');
+        form.action = "{{ url_for('ui.update_connection', connection_id='CONN_ID') }}".replace('CONN_ID', id);
+    });
+
+    const deleteModal = document.getElementById('deleteConnectionModal');
+    deleteModal.addEventListener('show.bs.modal', event => {
+        const button = event.relatedTarget;
+        const id = button.getAttribute('data-id');
+        const name = button.getAttribute('data-name');
+        
+        document.getElementById('deleteConnectionName').textContent = name;
+        const form = document.getElementById('deleteConnectionForm');
+        form.action = "{{ url_for('ui.delete_connection', connection_id='CONN_ID') }}".replace('CONN_ID', id);
+    });
+</script>
 {% endblock %}