3 Commits

Author SHA1 Message Date
97860669ec Fix presigned URL not generating for nested objects 2025-12-21 14:22:00 +08:00
4a5dd76286 Update installation and uninstallation scripts 2025-12-21 14:00:31 +08:00
d2dc293722 Fix inconsistency in config files 2025-12-21 13:17:33 +08:00
5 changed files with 342 additions and 148 deletions

View File

@@ -2,13 +2,14 @@
from __future__ import annotations
import logging
import shutil
import sys
import time
import uuid
from logging.handlers import RotatingFileHandler
from pathlib import Path
from datetime import timedelta
from typing import Any, Dict, Optional
from typing import Any, Dict, List, Optional
from flask import Flask, g, has_request_context, redirect, render_template, request, url_for
from flask_cors import CORS
@@ -28,6 +29,33 @@ from .storage import ObjectStorage
from .version import get_version
def _migrate_config_file(active_path: Path, legacy_paths: List[Path]) -> Path:
"""Migrate config file from legacy locations to the active path.
Checks each legacy path in order and moves the first one found to the active path.
This ensures backward compatibility for users upgrading from older versions.
"""
active_path.parent.mkdir(parents=True, exist_ok=True)
if active_path.exists():
return active_path
for legacy_path in legacy_paths:
if legacy_path.exists():
try:
shutil.move(str(legacy_path), str(active_path))
except OSError:
# Fall back to copy + delete if move fails (e.g., cross-device)
shutil.copy2(legacy_path, active_path)
try:
legacy_path.unlink(missing_ok=True)
except OSError:
pass
break
return active_path
def create_app(
test_config: Optional[Dict[str, Any]] = None,
*,
@@ -74,8 +102,26 @@ def create_app(
secret_store = EphemeralSecretStore(default_ttl=app.config.get("SECRET_TTL_SECONDS", 300))
# Initialize Replication components
connections_path = Path(app.config["STORAGE_ROOT"]) / ".connections.json"
replication_rules_path = Path(app.config["STORAGE_ROOT"]) / ".replication_rules.json"
# Store config files in the system config directory for consistency
storage_root = Path(app.config["STORAGE_ROOT"])
config_dir = storage_root / ".myfsio.sys" / "config"
config_dir.mkdir(parents=True, exist_ok=True)
# Define paths with migration from legacy locations
connections_path = _migrate_config_file(
active_path=config_dir / "connections.json",
legacy_paths=[
storage_root / ".myfsio.sys" / "connections.json", # Previous location
storage_root / ".connections.json", # Original legacy location
],
)
replication_rules_path = _migrate_config_file(
active_path=config_dir / "replication_rules.json",
legacy_paths=[
storage_root / ".myfsio.sys" / "replication_rules.json", # Previous location
storage_root / ".replication_rules.json", # Original legacy location
],
)
connections = ConnectionStore(connections_path)
replication = ReplicationManager(storage, connections, replication_rules_path)

View File

@@ -796,7 +796,7 @@ def object_presign(bucket_name: str, object_key: str):
api_base = current_app.config.get("API_BASE_URL") or "http://127.0.0.1:5000"
api_base = api_base.rstrip("/")
encoded_key = quote(object_key, safe="")
encoded_key = quote(object_key, safe="/")
url = f"{api_base}/presign/{bucket_name}/{encoded_key}"
# Use API base URL for forwarded headers so presigned URLs point to API, not UI

View File

@@ -1,7 +1,7 @@
"""Central location for the application version string."""
from __future__ import annotations
APP_VERSION = "0.1.5"
APP_VERSION = "0.1.6"
def get_version() -> str:

View File

@@ -4,8 +4,6 @@
# This script sets up MyFSIO for production use on Linux systems.
#
# Usage:
# curl -fsSL https://example.com/install.sh | bash
# OR
# ./install.sh [OPTIONS]
#
# Options:
@@ -23,14 +21,6 @@
set -e
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
# Default values
INSTALL_DIR="/opt/myfsio"
DATA_DIR="/var/lib/myfsio"
LOG_DIR="/var/log/myfsio"
@@ -42,7 +32,6 @@ SKIP_SYSTEMD=false
BINARY_PATH=""
AUTO_YES=false
# Parse arguments
while [[ $# -gt 0 ]]; do
case $1 in
--install-dir)
@@ -90,27 +79,30 @@ while [[ $# -gt 0 ]]; do
exit 0
;;
*)
echo -e "${RED}Unknown option: $1${NC}"
echo "Unknown option: $1"
exit 1
;;
esac
done
echo -e "${BLUE}"
echo "╔══════════════════════════════════════════════════════════╗"
echo " MyFSIO Installation "
echo " S3-Compatible Object Storage"
echo "╚══════════════════════════════════════════════════════════╝"
echo -e "${NC}"
echo ""
echo "============================================================"
echo " MyFSIO Installation Script"
echo " S3-Compatible Object Storage"
echo "============================================================"
echo ""
echo "Documentation: https://go.jzwsite.com/myfsio"
echo ""
# Check if running as root
if [[ $EUID -ne 0 ]]; then
echo -e "${RED}Error: This script must be run as root (use sudo)${NC}"
echo "Error: This script must be run as root (use sudo)"
exit 1
fi
# Display configuration
echo -e "${YELLOW}Installation Configuration:${NC}"
echo "------------------------------------------------------------"
echo "STEP 1: Review Installation Configuration"
echo "------------------------------------------------------------"
echo ""
echo " Install directory: $INSTALL_DIR"
echo " Data directory: $DATA_DIR"
echo " Log directory: $LOG_DIR"
@@ -125,9 +117,8 @@ if [[ -n "$BINARY_PATH" ]]; then
fi
echo ""
# Confirm installation
if [[ "$AUTO_YES" != true ]]; then
read -p "Proceed with installation? [y/N] " -n 1 -r
read -p "Do you want to proceed with these settings? [y/N] " -n 1 -r
echo
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
echo "Installation cancelled."
@@ -136,48 +127,70 @@ if [[ "$AUTO_YES" != true ]]; then
fi
echo ""
echo -e "${GREEN}[1/7]${NC} Creating system user..."
echo "------------------------------------------------------------"
echo "STEP 2: Creating System User"
echo "------------------------------------------------------------"
echo ""
if id "$SERVICE_USER" &>/dev/null; then
echo " User '$SERVICE_USER' already exists"
echo " [OK] User '$SERVICE_USER' already exists"
else
useradd --system --no-create-home --shell /usr/sbin/nologin "$SERVICE_USER"
echo " Created user '$SERVICE_USER'"
echo " [OK] Created user '$SERVICE_USER'"
fi
echo -e "${GREEN}[2/7]${NC} Creating directories..."
echo ""
echo "------------------------------------------------------------"
echo "STEP 3: Creating Directories"
echo "------------------------------------------------------------"
echo ""
mkdir -p "$INSTALL_DIR"
echo " [OK] Created $INSTALL_DIR"
mkdir -p "$DATA_DIR"
echo " [OK] Created $DATA_DIR"
mkdir -p "$LOG_DIR"
echo " Created $INSTALL_DIR"
echo " Created $DATA_DIR"
echo " Created $LOG_DIR"
echo " [OK] Created $LOG_DIR"
echo -e "${GREEN}[3/7]${NC} Installing binary..."
echo ""
echo "------------------------------------------------------------"
echo "STEP 4: Installing Binary"
echo "------------------------------------------------------------"
echo ""
if [[ -n "$BINARY_PATH" ]]; then
if [[ -f "$BINARY_PATH" ]]; then
cp "$BINARY_PATH" "$INSTALL_DIR/myfsio"
echo " Copied binary from $BINARY_PATH"
echo " [OK] Copied binary from $BINARY_PATH"
else
echo -e "${RED}Error: Binary not found at $BINARY_PATH${NC}"
echo " [ERROR] Binary not found at $BINARY_PATH"
exit 1
fi
elif [[ -f "./myfsio" ]]; then
cp "./myfsio" "$INSTALL_DIR/myfsio"
echo " Copied binary from ./myfsio"
echo " [OK] Copied binary from ./myfsio"
else
echo -e "${RED}Error: No binary provided. Use --binary PATH or place 'myfsio' in current directory${NC}"
echo " [ERROR] No binary provided."
echo " Use --binary PATH or place 'myfsio' in current directory"
exit 1
fi
chmod +x "$INSTALL_DIR/myfsio"
echo " [OK] Set executable permissions"
echo -e "${GREEN}[4/7]${NC} Generating secret key..."
echo ""
echo "------------------------------------------------------------"
echo "STEP 5: Generating Secret Key"
echo "------------------------------------------------------------"
echo ""
SECRET_KEY=$(openssl rand -base64 32)
echo " Generated secure SECRET_KEY"
echo " [OK] Generated secure SECRET_KEY"
echo -e "${GREEN}[5/7]${NC} Creating environment file..."
echo ""
echo "------------------------------------------------------------"
echo "STEP 6: Creating Configuration File"
echo "------------------------------------------------------------"
echo ""
cat > "$INSTALL_DIR/myfsio.env" << EOF
# MyFSIO Configuration
# Generated by install.sh on $(date)
# Documentation: https://go.jzwsite.com/myfsio
# Storage paths
STORAGE_ROOT=$DATA_DIR
@@ -206,20 +219,30 @@ RATE_LIMIT_DEFAULT=200 per minute
# KMS_ENABLED=true
EOF
chmod 600 "$INSTALL_DIR/myfsio.env"
echo " Created $INSTALL_DIR/myfsio.env"
echo " [OK] Created $INSTALL_DIR/myfsio.env"
echo -e "${GREEN}[6/7]${NC} Setting permissions..."
echo ""
echo "------------------------------------------------------------"
echo "STEP 7: Setting Permissions"
echo "------------------------------------------------------------"
echo ""
chown -R "$SERVICE_USER:$SERVICE_USER" "$INSTALL_DIR"
echo " [OK] Set ownership for $INSTALL_DIR"
chown -R "$SERVICE_USER:$SERVICE_USER" "$DATA_DIR"
echo " [OK] Set ownership for $DATA_DIR"
chown -R "$SERVICE_USER:$SERVICE_USER" "$LOG_DIR"
echo " Set ownership to $SERVICE_USER"
echo " [OK] Set ownership for $LOG_DIR"
if [[ "$SKIP_SYSTEMD" != true ]]; then
echo -e "${GREEN}[7/7]${NC} Creating systemd service..."
echo ""
echo "------------------------------------------------------------"
echo "STEP 8: Creating Systemd Service"
echo "------------------------------------------------------------"
echo ""
cat > /etc/systemd/system/myfsio.service << EOF
[Unit]
Description=MyFSIO S3-Compatible Storage
Documentation=https://github.com/yourusername/myfsio
Documentation=https://go.jzwsite.com/myfsio
After=network.target
[Service]
@@ -248,45 +271,100 @@ WantedBy=multi-user.target
EOF
systemctl daemon-reload
echo " Created /etc/systemd/system/myfsio.service"
echo " [OK] Created /etc/systemd/system/myfsio.service"
echo " [OK] Reloaded systemd daemon"
else
echo -e "${GREEN}[7/7]${NC} Skipping systemd service (--no-systemd)"
echo ""
echo "------------------------------------------------------------"
echo "STEP 8: Skipping Systemd Service (--no-systemd flag used)"
echo "------------------------------------------------------------"
fi
echo ""
echo -e "${GREEN}╔══════════════════════════════════════════════════════════╗${NC}"
echo -e "${GREEN} Installation Complete!${NC}"
echo -e "${GREEN}╚══════════════════════════════════════════════════════════╝${NC}"
echo "============================================================"
echo " Installation Complete!"
echo "============================================================"
echo ""
echo -e "${YELLOW}Next steps:${NC}"
if [[ "$SKIP_SYSTEMD" != true ]]; then
echo "------------------------------------------------------------"
echo "STEP 9: Start the Service"
echo "------------------------------------------------------------"
echo ""
echo " 1. Review configuration:"
echo " ${BLUE}cat $INSTALL_DIR/myfsio.env${NC}"
if [[ "$AUTO_YES" != true ]]; then
read -p "Would you like to start MyFSIO now? [Y/n] " -n 1 -r
echo
START_SERVICE=true
if [[ $REPLY =~ ^[Nn]$ ]]; then
START_SERVICE=false
fi
else
START_SERVICE=true
fi
if [[ "$START_SERVICE" == true ]]; then
echo " Starting MyFSIO service..."
systemctl start myfsio
echo " [OK] Service started"
echo ""
echo " 2. Start the service:"
echo " ${BLUE}sudo systemctl start myfsio${NC}"
read -p "Would you like to enable MyFSIO to start on boot? [Y/n] " -n 1 -r
echo
if [[ ! $REPLY =~ ^[Nn]$ ]]; then
systemctl enable myfsio
echo " [OK] Service enabled on boot"
fi
echo ""
echo " 3. Enable on boot:"
echo " ${BLUE}sudo systemctl enable myfsio${NC}"
sleep 2
echo " Service Status:"
echo " ---------------"
if systemctl is-active --quiet myfsio; then
echo " [OK] MyFSIO is running"
else
echo " [WARNING] MyFSIO may not have started correctly"
echo " Check logs with: journalctl -u myfsio -f"
fi
else
echo " [SKIPPED] Service not started"
echo ""
echo " 4. Check status:"
echo " ${BLUE}sudo systemctl status myfsio${NC}"
echo " To start manually, run:"
echo " sudo systemctl start myfsio"
echo ""
echo " 5. View logs:"
echo " ${BLUE}sudo journalctl -u myfsio -f${NC}"
echo " ${BLUE}tail -f $LOG_DIR/app.log${NC}"
echo " To enable on boot, run:"
echo " sudo systemctl enable myfsio"
fi
fi
echo ""
echo -e "${YELLOW}Access:${NC}"
echo " API: http://$(hostname -I | awk '{print $1}'):$API_PORT"
echo " UI: http://$(hostname -I | awk '{print $1}'):$UI_PORT/ui"
echo "============================================================"
echo " Summary"
echo "============================================================"
echo ""
echo -e "${YELLOW}Default credentials:${NC}"
echo "Access Points:"
echo " API: http://$(hostname -I 2>/dev/null | awk '{print $1}' || echo "localhost"):$API_PORT"
echo " UI: http://$(hostname -I 2>/dev/null | awk '{print $1}' || echo "localhost"):$UI_PORT/ui"
echo ""
echo "Default Credentials:"
echo " Username: localadmin"
echo " Password: localadmin"
echo -e " ${RED} Change these immediately after first login!${NC}"
echo " [!] WARNING: Change these immediately after first login!"
echo ""
echo -e "${YELLOW}Configuration files:${NC}"
echo "Configuration Files:"
echo " Environment: $INSTALL_DIR/myfsio.env"
echo " IAM Users: $DATA_DIR/.myfsio.sys/config/iam.json"
echo " Bucket Policies: $DATA_DIR/.myfsio.sys/config/bucket_policies.json"
echo ""
echo "Useful Commands:"
echo " Check status: sudo systemctl status myfsio"
echo " View logs: sudo journalctl -u myfsio -f"
echo " Restart: sudo systemctl restart myfsio"
echo " Stop: sudo systemctl stop myfsio"
echo ""
echo "Documentation: https://go.jzwsite.com/myfsio"
echo ""
echo "============================================================"
echo " Thank you for installing MyFSIO!"
echo "============================================================"
echo ""

View File

@@ -18,13 +18,6 @@
set -e
# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'
# Default values
INSTALL_DIR="/opt/myfsio"
DATA_DIR="/var/lib/myfsio"
LOG_DIR="/var/log/myfsio"
@@ -33,7 +26,6 @@ KEEP_DATA=false
KEEP_LOGS=false
AUTO_YES=false
# Parse arguments
while [[ $# -gt 0 ]]; do
case $1 in
--keep-data)
@@ -69,106 +61,184 @@ while [[ $# -gt 0 ]]; do
exit 0
;;
*)
echo -e "${RED}Unknown option: $1${NC}"
echo "Unknown option: $1"
exit 1
;;
esac
done
echo -e "${RED}"
echo "╔══════════════════════════════════════════════════════════╗"
echo " MyFSIO Uninstallation "
echo "╚══════════════════════════════════════════════════════════╝"
echo -e "${NC}"
echo ""
echo "============================================================"
echo " MyFSIO Uninstallation Script"
echo "============================================================"
echo ""
echo "Documentation: https://go.jzwsite.com/myfsio"
echo ""
# Check if running as root
if [[ $EUID -ne 0 ]]; then
echo -e "${RED}Error: This script must be run as root (use sudo)${NC}"
echo "Error: This script must be run as root (use sudo)"
exit 1
fi
echo -e "${YELLOW}The following will be removed:${NC}"
echo "------------------------------------------------------------"
echo "STEP 1: Review What Will Be Removed"
echo "------------------------------------------------------------"
echo ""
echo "The following items will be removed:"
echo ""
echo " Install directory: $INSTALL_DIR"
if [[ "$KEEP_DATA" != true ]]; then
echo -e " Data directory: $DATA_DIR ${RED}(ALL YOUR DATA!)${NC}"
echo " Data directory: $DATA_DIR (ALL YOUR DATA WILL BE DELETED!)"
else
echo " Data directory: $DATA_DIR (KEPT)"
echo " Data directory: $DATA_DIR (WILL BE KEPT)"
fi
if [[ "$KEEP_LOGS" != true ]]; then
echo " Log directory: $LOG_DIR"
else
echo " Log directory: $LOG_DIR (KEPT)"
echo " Log directory: $LOG_DIR (WILL BE KEPT)"
fi
echo " Systemd service: /etc/systemd/system/myfsio.service"
echo " System user: $SERVICE_USER"
echo ""
if [[ "$AUTO_YES" != true ]]; then
echo -e "${RED}WARNING: This action cannot be undone!${NC}"
echo "WARNING: This action cannot be undone!"
echo ""
read -p "Are you sure you want to uninstall MyFSIO? [y/N] " -n 1 -r
echo
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
echo ""
echo "Uninstallation cancelled."
exit 0
fi
if [[ "$KEEP_DATA" != true ]]; then
echo ""
read -p "This will DELETE ALL YOUR DATA. Type 'DELETE' to confirm: " CONFIRM
if [[ "$CONFIRM" != "DELETE" ]]; then
echo ""
echo "Uninstallation cancelled."
echo "Tip: Use --keep-data to preserve your data directory"
exit 0
fi
fi
fi
echo ""
echo -e "${GREEN}[1/5]${NC} Stopping service..."
echo "------------------------------------------------------------"
echo "STEP 2: Stopping Service"
echo "------------------------------------------------------------"
echo ""
if systemctl is-active --quiet myfsio 2>/dev/null; then
systemctl stop myfsio
echo " Stopped myfsio service"
echo " [OK] Stopped myfsio service"
else
echo " Service not running"
echo " [SKIP] Service not running"
fi
echo -e "${GREEN}[2/5]${NC} Disabling service..."
echo ""
echo "------------------------------------------------------------"
echo "STEP 3: Disabling Service"
echo "------------------------------------------------------------"
echo ""
if systemctl is-enabled --quiet myfsio 2>/dev/null; then
systemctl disable myfsio
echo " Disabled myfsio service"
echo " [OK] Disabled myfsio service"
else
echo " Service not enabled"
echo " [SKIP] Service not enabled"
fi
echo -e "${GREEN}[3/5]${NC} Removing systemd service..."
echo ""
echo "------------------------------------------------------------"
echo "STEP 4: Removing Systemd Service File"
echo "------------------------------------------------------------"
echo ""
if [[ -f /etc/systemd/system/myfsio.service ]]; then
rm -f /etc/systemd/system/myfsio.service
systemctl daemon-reload
echo " Removed /etc/systemd/system/myfsio.service"
echo " [OK] Removed /etc/systemd/system/myfsio.service"
echo " [OK] Reloaded systemd daemon"
else
echo " Service file not found"
fi
echo -e "${GREEN}[4/5]${NC} Removing directories..."
if [[ -d "$INSTALL_DIR" ]]; then
rm -rf "$INSTALL_DIR"
echo " Removed $INSTALL_DIR"
fi
if [[ "$KEEP_DATA" != true ]] && [[ -d "$DATA_DIR" ]]; then
rm -rf "$DATA_DIR"
echo " Removed $DATA_DIR"
elif [[ "$KEEP_DATA" == true ]]; then
echo " Kept $DATA_DIR"
fi
if [[ "$KEEP_LOGS" != true ]] && [[ -d "$LOG_DIR" ]]; then
rm -rf "$LOG_DIR"
echo " Removed $LOG_DIR"
elif [[ "$KEEP_LOGS" == true ]]; then
echo " Kept $LOG_DIR"
fi
echo -e "${GREEN}[5/5]${NC} Removing system user..."
if id "$SERVICE_USER" &>/dev/null; then
userdel "$SERVICE_USER" 2>/dev/null || true
echo " Removed user '$SERVICE_USER'"
else
echo " User not found"
echo " [SKIP] Service file not found"
fi
echo ""
echo -e "${GREEN}MyFSIO has been uninstalled.${NC}"
if [[ "$KEEP_DATA" == true ]]; then
echo -e "${YELLOW}Data preserved at: $DATA_DIR${NC}"
echo "------------------------------------------------------------"
echo "STEP 5: Removing Installation Directory"
echo "------------------------------------------------------------"
echo ""
if [[ -d "$INSTALL_DIR" ]]; then
rm -rf "$INSTALL_DIR"
echo " [OK] Removed $INSTALL_DIR"
else
echo " [SKIP] Directory not found: $INSTALL_DIR"
fi
echo ""
echo "------------------------------------------------------------"
echo "STEP 6: Removing Data Directory"
echo "------------------------------------------------------------"
echo ""
if [[ "$KEEP_DATA" != true ]]; then
if [[ -d "$DATA_DIR" ]]; then
rm -rf "$DATA_DIR"
echo " [OK] Removed $DATA_DIR"
else
echo " [SKIP] Directory not found: $DATA_DIR"
fi
else
echo " [KEPT] Data preserved at: $DATA_DIR"
fi
echo ""
echo "------------------------------------------------------------"
echo "STEP 7: Removing Log Directory"
echo "------------------------------------------------------------"
echo ""
if [[ "$KEEP_LOGS" != true ]]; then
if [[ -d "$LOG_DIR" ]]; then
rm -rf "$LOG_DIR"
echo " [OK] Removed $LOG_DIR"
else
echo " [SKIP] Directory not found: $LOG_DIR"
fi
else
echo " [KEPT] Logs preserved at: $LOG_DIR"
fi
echo ""
echo "------------------------------------------------------------"
echo "STEP 8: Removing System User"
echo "------------------------------------------------------------"
echo ""
if id "$SERVICE_USER" &>/dev/null; then
userdel "$SERVICE_USER" 2>/dev/null || true
echo " [OK] Removed user '$SERVICE_USER'"
else
echo " [SKIP] User not found: $SERVICE_USER"
fi
echo ""
echo "============================================================"
echo " Uninstallation Complete!"
echo "============================================================"
echo ""
if [[ "$KEEP_DATA" == true ]]; then
echo "Your data has been preserved at: $DATA_DIR"
echo ""
echo "To reinstall MyFSIO with existing data, run:"
echo " curl -fsSL https://go.jzwsite.com/myfsio-install | sudo bash"
echo ""
fi
if [[ "$KEEP_LOGS" == true ]]; then
echo "Your logs have been preserved at: $LOG_DIR"
echo ""
fi
echo "Thank you for using MyFSIO."
echo "Documentation: https://go.jzwsite.com/myfsio"
echo ""
echo "============================================================"
echo ""