Compare commits
99 Commits
v0.2.0
...
4d90ead816
| Author | SHA1 | Date | |
|---|---|---|---|
| 4d90ead816 | |||
| 20a314e030 | |||
| b37a51ed1d | |||
| d8232340c3 | |||
| a356bb0c4e | |||
| 1c328ee3af | |||
| 5bf7962c04 | |||
| e06f653606 | |||
| 0462a7b62e | |||
| 9c2809c195 | |||
| fb32ca0a7d | |||
| 6ab702a818 | |||
| 550e7d435c | |||
| 776967e80d | |||
| 082a7fbcd1 | |||
| ff287cf67b | |||
| bddf36d52d | |||
| cf6cec9cab | |||
| d425839e57 | |||
| 4c661477d5 | |||
| f3f52f14a5 | |||
| d19ba3e305 | |||
| c627f41f53 | |||
| bcad0cd3da | |||
| 52660570c1 | |||
| 67f057ca1c | |||
| 35f61313e0 | |||
| 01e79e6993 | |||
| 1e3c4b545f | |||
| c470cfb576 | |||
| 4ecd32a554 | |||
| aa6d7c4d28 | |||
| 6e6d6d32bf | |||
| 54705ab9c4 | |||
| d96955deee | |||
| 77a46d0725 | |||
| 0f750b9d89 | |||
| e0dee9db36 | |||
| 126657c99f | |||
| 07fb1ac773 | |||
| 147962e1dd | |||
| 2643a79121 | |||
| e9a035827b | |||
| 033b8a82be | |||
| e76c311231 | |||
| cbdf1a27c8 | |||
| 4a60cb269a | |||
| ebe7f6222d | |||
| 70b61fd8e6 | |||
| 85181f0be6 | |||
| a779b002d7 | |||
| d5ca7a8be1 | |||
| 45d21cce21 | |||
| 9629507acd | |||
| 5d6cb4efa1 | |||
| 56ad83bbaf | |||
| 847933b7c0 | |||
| be55d08c0a | |||
| 8c4bf67974 | |||
| 9385d1fe1c | |||
| 0ea54457e8 | |||
| ae26d22388 | |||
| 6b715851b9 | |||
| 62c36f7a6c | |||
| b32f1f94f7 | |||
| 6e3d280a75 | |||
| 704f79dc44 | |||
| 476dc79e42 | |||
| 87c7f1bc7d | |||
| 23ea164215 | |||
| 7a8acfb933 | |||
| 71327bcbf1 | |||
| c0603c592b | |||
| 912a7dc74f | |||
| bb6590fc5e | |||
| 4de936cea9 | |||
| adb9017580 | |||
| 4adfcc4131 | |||
| ebc315c1cc | |||
| 5ab62a00ff | |||
| 9c3518de63 | |||
| a52657e684 | |||
| 53297abe1e | |||
| a3b9db544c | |||
| f5d2e1c488 | |||
| f04c6a9cdc | |||
| 7a494abb96 | |||
| 956d17a649 | |||
| 5522f9ac04 | |||
| 3742f0228e | |||
| ba694cb717 | |||
| 433d291b4b | |||
| 899db3421b | |||
| e3509e997f | |||
| 1c30200db0 | |||
| 7ff422d4dc | |||
| 546d51af9a | |||
| 0d1fe05fd0 | |||
| c5d4b2f1cd |
4
.gitignore
vendored
4
.gitignore
vendored
@@ -26,6 +26,10 @@ dist/
|
||||
*.egg-info/
|
||||
.eggs/
|
||||
|
||||
# Rust / maturin build artifacts
|
||||
myfsio_core/target/
|
||||
myfsio_core/Cargo.lock
|
||||
|
||||
# Local runtime artifacts
|
||||
logs/
|
||||
*.log
|
||||
|
||||
24
Dockerfile
24
Dockerfile
@@ -1,25 +1,33 @@
|
||||
# syntax=docker/dockerfile:1.7
|
||||
FROM python:3.12.12-slim
|
||||
FROM python:3.14.3-slim
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Install build deps for any wheels that need compilation, then clean up
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends build-essential \
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends build-essential curl \
|
||||
&& curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
ENV PATH="/root/.cargo/bin:${PATH}"
|
||||
|
||||
COPY requirements.txt ./
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY . .
|
||||
|
||||
# Make entrypoint executable
|
||||
RUN pip install --no-cache-dir maturin \
|
||||
&& cd myfsio_core \
|
||||
&& maturin build --release \
|
||||
&& pip install target/wheels/*.whl \
|
||||
&& cd .. \
|
||||
&& rm -rf myfsio_core/target \
|
||||
&& pip uninstall -y maturin \
|
||||
&& rustup self uninstall -y
|
||||
|
||||
RUN chmod +x docker-entrypoint.sh
|
||||
|
||||
# Create data directory and set permissions
|
||||
RUN mkdir -p /app/data \
|
||||
&& useradd -m -u 1000 myfsio \
|
||||
&& chown -R myfsio:myfsio /app
|
||||
@@ -32,6 +40,6 @@ ENV APP_HOST=0.0.0.0 \
|
||||
FLASK_DEBUG=0
|
||||
|
||||
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
|
||||
CMD python -c "import requests; requests.get('http://localhost:5000/healthz', timeout=2)"
|
||||
CMD python -c "import requests; requests.get('http://localhost:5000/myfsio/health', timeout=2)"
|
||||
|
||||
CMD ["./docker-entrypoint.sh"]
|
||||
|
||||
661
LICENSE
Normal file
661
LICENSE
Normal file
@@ -0,0 +1,661 @@
|
||||
GNU AFFERO GENERAL PUBLIC LICENSE
|
||||
Version 3, 19 November 2007
|
||||
|
||||
Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
|
||||
Everyone is permitted to copy and distribute verbatim copies
|
||||
of this license document, but changing it is not allowed.
|
||||
|
||||
Preamble
|
||||
|
||||
The GNU Affero General Public License is a free, copyleft license for
|
||||
software and other kinds of works, specifically designed to ensure
|
||||
cooperation with the community in the case of network server software.
|
||||
|
||||
The licenses for most software and other practical works are designed
|
||||
to take away your freedom to share and change the works. By contrast,
|
||||
our General Public Licenses are intended to guarantee your freedom to
|
||||
share and change all versions of a program--to make sure it remains free
|
||||
software for all its users.
|
||||
|
||||
When we speak of free software, we are referring to freedom, not
|
||||
price. Our General Public Licenses are designed to make sure that you
|
||||
have the freedom to distribute copies of free software (and charge for
|
||||
them if you wish), that you receive source code or can get it if you
|
||||
want it, that you can change the software or use pieces of it in new
|
||||
free programs, and that you know you can do these things.
|
||||
|
||||
Developers that use our General Public Licenses protect your rights
|
||||
with two steps: (1) assert copyright on the software, and (2) offer
|
||||
you this License which gives you legal permission to copy, distribute
|
||||
and/or modify the software.
|
||||
|
||||
A secondary benefit of defending all users' freedom is that
|
||||
improvements made in alternate versions of the program, if they
|
||||
receive widespread use, become available for other developers to
|
||||
incorporate. Many developers of free software are heartened and
|
||||
encouraged by the resulting cooperation. However, in the case of
|
||||
software used on network servers, this result may fail to come about.
|
||||
The GNU General Public License permits making a modified version and
|
||||
letting the public access it on a server without ever releasing its
|
||||
source code to the public.
|
||||
|
||||
The GNU Affero General Public License is designed specifically to
|
||||
ensure that, in such cases, the modified source code becomes available
|
||||
to the community. It requires the operator of a network server to
|
||||
provide the source code of the modified version running there to the
|
||||
users of that server. Therefore, public use of a modified version, on
|
||||
a publicly accessible server, gives the public access to the source
|
||||
code of the modified version.
|
||||
|
||||
An older license, called the Affero General Public License and
|
||||
published by Affero, was designed to accomplish similar goals. This is
|
||||
a different license, not a version of the Affero GPL, but Affero has
|
||||
released a new version of the Affero GPL which permits relicensing under
|
||||
this license.
|
||||
|
||||
The precise terms and conditions for copying, distribution and
|
||||
modification follow.
|
||||
|
||||
TERMS AND CONDITIONS
|
||||
|
||||
0. Definitions.
|
||||
|
||||
"This License" refers to version 3 of the GNU Affero General Public License.
|
||||
|
||||
"Copyright" also means copyright-like laws that apply to other kinds of
|
||||
works, such as semiconductor masks.
|
||||
|
||||
"The Program" refers to any copyrightable work licensed under this
|
||||
License. Each licensee is addressed as "you". "Licensees" and
|
||||
"recipients" may be individuals or organizations.
|
||||
|
||||
To "modify" a work means to copy from or adapt all or part of the work
|
||||
in a fashion requiring copyright permission, other than the making of an
|
||||
exact copy. The resulting work is called a "modified version" of the
|
||||
earlier work or a work "based on" the earlier work.
|
||||
|
||||
A "covered work" means either the unmodified Program or a work based
|
||||
on the Program.
|
||||
|
||||
To "propagate" a work means to do anything with it that, without
|
||||
permission, would make you directly or secondarily liable for
|
||||
infringement under applicable copyright law, except executing it on a
|
||||
computer or modifying a private copy. Propagation includes copying,
|
||||
distribution (with or without modification), making available to the
|
||||
public, and in some countries other activities as well.
|
||||
|
||||
To "convey" a work means any kind of propagation that enables other
|
||||
parties to make or receive copies. Mere interaction with a user through
|
||||
a computer network, with no transfer of a copy, is not conveying.
|
||||
|
||||
An interactive user interface displays "Appropriate Legal Notices"
|
||||
to the extent that it includes a convenient and prominently visible
|
||||
feature that (1) displays an appropriate copyright notice, and (2)
|
||||
tells the user that there is no warranty for the work (except to the
|
||||
extent that warranties are provided), that licensees may convey the
|
||||
work under this License, and how to view a copy of this License. If
|
||||
the interface presents a list of user commands or options, such as a
|
||||
menu, a prominent item in the list meets this criterion.
|
||||
|
||||
1. Source Code.
|
||||
|
||||
The "source code" for a work means the preferred form of the work
|
||||
for making modifications to it. "Object code" means any non-source
|
||||
form of a work.
|
||||
|
||||
A "Standard Interface" means an interface that either is an official
|
||||
standard defined by a recognized standards body, or, in the case of
|
||||
interfaces specified for a particular programming language, one that
|
||||
is widely used among developers working in that language.
|
||||
|
||||
The "System Libraries" of an executable work include anything, other
|
||||
than the work as a whole, that (a) is included in the normal form of
|
||||
packaging a Major Component, but which is not part of that Major
|
||||
Component, and (b) serves only to enable use of the work with that
|
||||
Major Component, or to implement a Standard Interface for which an
|
||||
implementation is available to the public in source code form. A
|
||||
"Major Component", in this context, means a major essential component
|
||||
(kernel, window system, and so on) of the specific operating system
|
||||
(if any) on which the executable work runs, or a compiler used to
|
||||
produce the work, or an object code interpreter used to run it.
|
||||
|
||||
The "Corresponding Source" for a work in object code form means all
|
||||
the source code needed to generate, install, and (for an executable
|
||||
work) run the object code and to modify the work, including scripts to
|
||||
control those activities. However, it does not include the work's
|
||||
System Libraries, or general-purpose tools or generally available free
|
||||
programs which are used unmodified in performing those activities but
|
||||
which are not part of the work. For example, Corresponding Source
|
||||
includes interface definition files associated with source files for
|
||||
the work, and the source code for shared libraries and dynamically
|
||||
linked subprograms that the work is specifically designed to require,
|
||||
such as by intimate data communication or control flow between those
|
||||
subprograms and other parts of the work.
|
||||
|
||||
The Corresponding Source need not include anything that users
|
||||
can regenerate automatically from other parts of the Corresponding
|
||||
Source.
|
||||
|
||||
The Corresponding Source for a work in source code form is that
|
||||
same work.
|
||||
|
||||
2. Basic Permissions.
|
||||
|
||||
All rights granted under this License are granted for the term of
|
||||
copyright on the Program, and are irrevocable provided the stated
|
||||
conditions are met. This License explicitly affirms your unlimited
|
||||
permission to run the unmodified Program. The output from running a
|
||||
covered work is covered by this License only if the output, given its
|
||||
content, constitutes a covered work. This License acknowledges your
|
||||
rights of fair use or other equivalent, as provided by copyright law.
|
||||
|
||||
You may make, run and propagate covered works that you do not
|
||||
convey, without conditions so long as your license otherwise remains
|
||||
in force. You may convey covered works to others for the sole purpose
|
||||
of having them make modifications exclusively for you, or provide you
|
||||
with facilities for running those works, provided that you comply with
|
||||
the terms of this License in conveying all material for which you do
|
||||
not control copyright. Those thus making or running the covered works
|
||||
for you must do so exclusively on your behalf, under your direction
|
||||
and control, on terms that prohibit them from making any copies of
|
||||
your copyrighted material outside their relationship with you.
|
||||
|
||||
Conveying under any other circumstances is permitted solely under
|
||||
the conditions stated below. Sublicensing is not allowed; section 10
|
||||
makes it unnecessary.
|
||||
|
||||
3. Protecting Users' Legal Rights From Anti-Circumvention Law.
|
||||
|
||||
No covered work shall be deemed part of an effective technological
|
||||
measure under any applicable law fulfilling obligations under article
|
||||
11 of the WIPO copyright treaty adopted on 20 December 1996, or
|
||||
similar laws prohibiting or restricting circumvention of such
|
||||
measures.
|
||||
|
||||
When you convey a covered work, you waive any legal power to forbid
|
||||
circumvention of technological measures to the extent such circumvention
|
||||
is effected by exercising rights under this License with respect to
|
||||
the covered work, and you disclaim any intention to limit operation or
|
||||
modification of the work as a means of enforcing, against the work's
|
||||
users, your or third parties' legal rights to forbid circumvention of
|
||||
technological measures.
|
||||
|
||||
4. Conveying Verbatim Copies.
|
||||
|
||||
You may convey verbatim copies of the Program's source code as you
|
||||
receive it, in any medium, provided that you conspicuously and
|
||||
appropriately publish on each copy an appropriate copyright notice;
|
||||
keep intact all notices stating that this License and any
|
||||
non-permissive terms added in accord with section 7 apply to the code;
|
||||
keep intact all notices of the absence of any warranty; and give all
|
||||
recipients a copy of this License along with the Program.
|
||||
|
||||
You may charge any price or no price for each copy that you convey,
|
||||
and you may offer support or warranty protection for a fee.
|
||||
|
||||
5. Conveying Modified Source Versions.
|
||||
|
||||
You may convey a work based on the Program, or the modifications to
|
||||
produce it from the Program, in the form of source code under the
|
||||
terms of section 4, provided that you also meet all of these conditions:
|
||||
|
||||
a) The work must carry prominent notices stating that you modified
|
||||
it, and giving a relevant date.
|
||||
|
||||
b) The work must carry prominent notices stating that it is
|
||||
released under this License and any conditions added under section
|
||||
7. This requirement modifies the requirement in section 4 to
|
||||
"keep intact all notices".
|
||||
|
||||
c) You must license the entire work, as a whole, under this
|
||||
License to anyone who comes into possession of a copy. This
|
||||
License will therefore apply, along with any applicable section 7
|
||||
additional terms, to the whole of the work, and all its parts,
|
||||
regardless of how they are packaged. This License gives no
|
||||
permission to license the work in any other way, but it does not
|
||||
invalidate such permission if you have separately received it.
|
||||
|
||||
d) If the work has interactive user interfaces, each must display
|
||||
Appropriate Legal Notices; however, if the Program has interactive
|
||||
interfaces that do not display Appropriate Legal Notices, your
|
||||
work need not make them do so.
|
||||
|
||||
A compilation of a covered work with other separate and independent
|
||||
works, which are not by their nature extensions of the covered work,
|
||||
and which are not combined with it such as to form a larger program,
|
||||
in or on a volume of a storage or distribution medium, is called an
|
||||
"aggregate" if the compilation and its resulting copyright are not
|
||||
used to limit the access or legal rights of the compilation's users
|
||||
beyond what the individual works permit. Inclusion of a covered work
|
||||
in an aggregate does not cause this License to apply to the other
|
||||
parts of the aggregate.
|
||||
|
||||
6. Conveying Non-Source Forms.
|
||||
|
||||
You may convey a covered work in object code form under the terms
|
||||
of sections 4 and 5, provided that you also convey the
|
||||
machine-readable Corresponding Source under the terms of this License,
|
||||
in one of these ways:
|
||||
|
||||
a) Convey the object code in, or embodied in, a physical product
|
||||
(including a physical distribution medium), accompanied by the
|
||||
Corresponding Source fixed on a durable physical medium
|
||||
customarily used for software interchange.
|
||||
|
||||
b) Convey the object code in, or embodied in, a physical product
|
||||
(including a physical distribution medium), accompanied by a
|
||||
written offer, valid for at least three years and valid for as
|
||||
long as you offer spare parts or customer support for that product
|
||||
model, to give anyone who possesses the object code either (1) a
|
||||
copy of the Corresponding Source for all the software in the
|
||||
product that is covered by this License, on a durable physical
|
||||
medium customarily used for software interchange, for a price no
|
||||
more than your reasonable cost of physically performing this
|
||||
conveying of source, or (2) access to copy the
|
||||
Corresponding Source from a network server at no charge.
|
||||
|
||||
c) Convey individual copies of the object code with a copy of the
|
||||
written offer to provide the Corresponding Source. This
|
||||
alternative is allowed only occasionally and noncommercially, and
|
||||
only if you received the object code with such an offer, in accord
|
||||
with subsection 6b.
|
||||
|
||||
d) Convey the object code by offering access from a designated
|
||||
place (gratis or for a charge), and offer equivalent access to the
|
||||
Corresponding Source in the same way through the same place at no
|
||||
further charge. You need not require recipients to copy the
|
||||
Corresponding Source along with the object code. If the place to
|
||||
copy the object code is a network server, the Corresponding Source
|
||||
may be on a different server (operated by you or a third party)
|
||||
that supports equivalent copying facilities, provided you maintain
|
||||
clear directions next to the object code saying where to find the
|
||||
Corresponding Source. Regardless of what server hosts the
|
||||
Corresponding Source, you remain obligated to ensure that it is
|
||||
available for as long as needed to satisfy these requirements.
|
||||
|
||||
e) Convey the object code using peer-to-peer transmission, provided
|
||||
you inform other peers where the object code and Corresponding
|
||||
Source of the work are being offered to the general public at no
|
||||
charge under subsection 6d.
|
||||
|
||||
A separable portion of the object code, whose source code is excluded
|
||||
from the Corresponding Source as a System Library, need not be
|
||||
included in conveying the object code work.
|
||||
|
||||
A "User Product" is either (1) a "consumer product", which means any
|
||||
tangible personal property which is normally used for personal, family,
|
||||
or household purposes, or (2) anything designed or sold for incorporation
|
||||
into a dwelling. In determining whether a product is a consumer product,
|
||||
doubtful cases shall be resolved in favor of coverage. For a particular
|
||||
product received by a particular user, "normally used" refers to a
|
||||
typical or common use of that class of product, regardless of the status
|
||||
of the particular user or of the way in which the particular user
|
||||
actually uses, or expects or is expected to use, the product. A product
|
||||
is a consumer product regardless of whether the product has substantial
|
||||
commercial, industrial or non-consumer uses, unless such uses represent
|
||||
the only significant mode of use of the product.
|
||||
|
||||
"Installation Information" for a User Product means any methods,
|
||||
procedures, authorization keys, or other information required to install
|
||||
and execute modified versions of a covered work in that User Product from
|
||||
a modified version of its Corresponding Source. The information must
|
||||
suffice to ensure that the continued functioning of the modified object
|
||||
code is in no case prevented or interfered with solely because
|
||||
modification has been made.
|
||||
|
||||
If you convey an object code work under this section in, or with, or
|
||||
specifically for use in, a User Product, and the conveying occurs as
|
||||
part of a transaction in which the right of possession and use of the
|
||||
User Product is transferred to the recipient in perpetuity or for a
|
||||
fixed term (regardless of how the transaction is characterized), the
|
||||
Corresponding Source conveyed under this section must be accompanied
|
||||
by the Installation Information. But this requirement does not apply
|
||||
if neither you nor any third party retains the ability to install
|
||||
modified object code on the User Product (for example, the work has
|
||||
been installed in ROM).
|
||||
|
||||
The requirement to provide Installation Information does not include a
|
||||
requirement to continue to provide support service, warranty, or updates
|
||||
for a work that has been modified or installed by the recipient, or for
|
||||
the User Product in which it has been modified or installed. Access to a
|
||||
network may be denied when the modification itself materially and
|
||||
adversely affects the operation of the network or violates the rules and
|
||||
protocols for communication across the network.
|
||||
|
||||
Corresponding Source conveyed, and Installation Information provided,
|
||||
in accord with this section must be in a format that is publicly
|
||||
documented (and with an implementation available to the public in
|
||||
source code form), and must require no special password or key for
|
||||
unpacking, reading or copying.
|
||||
|
||||
7. Additional Terms.
|
||||
|
||||
"Additional permissions" are terms that supplement the terms of this
|
||||
License by making exceptions from one or more of its conditions.
|
||||
Additional permissions that are applicable to the entire Program shall
|
||||
be treated as though they were included in this License, to the extent
|
||||
that they are valid under applicable law. If additional permissions
|
||||
apply only to part of the Program, that part may be used separately
|
||||
under those permissions, but the entire Program remains governed by
|
||||
this License without regard to the additional permissions.
|
||||
|
||||
When you convey a copy of a covered work, you may at your option
|
||||
remove any additional permissions from that copy, or from any part of
|
||||
it. (Additional permissions may be written to require their own
|
||||
removal in certain cases when you modify the work.) You may place
|
||||
additional permissions on material, added by you to a covered work,
|
||||
for which you have or can give appropriate copyright permission.
|
||||
|
||||
Notwithstanding any other provision of this License, for material you
|
||||
add to a covered work, you may (if authorized by the copyright holders of
|
||||
that material) supplement the terms of this License with terms:
|
||||
|
||||
a) Disclaiming warranty or limiting liability differently from the
|
||||
terms of sections 15 and 16 of this License; or
|
||||
|
||||
b) Requiring preservation of specified reasonable legal notices or
|
||||
author attributions in that material or in the Appropriate Legal
|
||||
Notices displayed by works containing it; or
|
||||
|
||||
c) Prohibiting misrepresentation of the origin of that material, or
|
||||
requiring that modified versions of such material be marked in
|
||||
reasonable ways as different from the original version; or
|
||||
|
||||
d) Limiting the use for publicity purposes of names of licensors or
|
||||
authors of the material; or
|
||||
|
||||
e) Declining to grant rights under trademark law for use of some
|
||||
trade names, trademarks, or service marks; or
|
||||
|
||||
f) Requiring indemnification of licensors and authors of that
|
||||
material by anyone who conveys the material (or modified versions of
|
||||
it) with contractual assumptions of liability to the recipient, for
|
||||
any liability that these contractual assumptions directly impose on
|
||||
those licensors and authors.
|
||||
|
||||
All other non-permissive additional terms are considered "further
|
||||
restrictions" within the meaning of section 10. If the Program as you
|
||||
received it, or any part of it, contains a notice stating that it is
|
||||
governed by this License along with a term that is a further
|
||||
restriction, you may remove that term. If a license document contains
|
||||
a further restriction but permits relicensing or conveying under this
|
||||
License, you may add to a covered work material governed by the terms
|
||||
of that license document, provided that the further restriction does
|
||||
not survive such relicensing or conveying.
|
||||
|
||||
If you add terms to a covered work in accord with this section, you
|
||||
must place, in the relevant source files, a statement of the
|
||||
additional terms that apply to those files, or a notice indicating
|
||||
where to find the applicable terms.
|
||||
|
||||
Additional terms, permissive or non-permissive, may be stated in the
|
||||
form of a separately written license, or stated as exceptions;
|
||||
the above requirements apply either way.
|
||||
|
||||
8. Termination.
|
||||
|
||||
You may not propagate or modify a covered work except as expressly
|
||||
provided under this License. Any attempt otherwise to propagate or
|
||||
modify it is void, and will automatically terminate your rights under
|
||||
this License (including any patent licenses granted under the third
|
||||
paragraph of section 11).
|
||||
|
||||
However, if you cease all violation of this License, then your
|
||||
license from a particular copyright holder is reinstated (a)
|
||||
provisionally, unless and until the copyright holder explicitly and
|
||||
finally terminates your license, and (b) permanently, if the copyright
|
||||
holder fails to notify you of the violation by some reasonable means
|
||||
prior to 60 days after the cessation.
|
||||
|
||||
Moreover, your license from a particular copyright holder is
|
||||
reinstated permanently if the copyright holder notifies you of the
|
||||
violation by some reasonable means, this is the first time you have
|
||||
received notice of violation of this License (for any work) from that
|
||||
copyright holder, and you cure the violation prior to 30 days after
|
||||
your receipt of the notice.
|
||||
|
||||
Termination of your rights under this section does not terminate the
|
||||
licenses of parties who have received copies or rights from you under
|
||||
this License. If your rights have been terminated and not permanently
|
||||
reinstated, you do not qualify to receive new licenses for the same
|
||||
material under section 10.
|
||||
|
||||
9. Acceptance Not Required for Having Copies.
|
||||
|
||||
You are not required to accept this License in order to receive or
|
||||
run a copy of the Program. Ancillary propagation of a covered work
|
||||
occurring solely as a consequence of using peer-to-peer transmission
|
||||
to receive a copy likewise does not require acceptance. However,
|
||||
nothing other than this License grants you permission to propagate or
|
||||
modify any covered work. These actions infringe copyright if you do
|
||||
not accept this License. Therefore, by modifying or propagating a
|
||||
covered work, you indicate your acceptance of this License to do so.
|
||||
|
||||
10. Automatic Licensing of Downstream Recipients.
|
||||
|
||||
Each time you convey a covered work, the recipient automatically
|
||||
receives a license from the original licensors, to run, modify and
|
||||
propagate that work, subject to this License. You are not responsible
|
||||
for enforcing compliance by third parties with this License.
|
||||
|
||||
An "entity transaction" is a transaction transferring control of an
|
||||
organization, or substantially all assets of one, or subdividing an
|
||||
organization, or merging organizations. If propagation of a covered
|
||||
work results from an entity transaction, each party to that
|
||||
transaction who receives a copy of the work also receives whatever
|
||||
licenses to the work the party's predecessor in interest had or could
|
||||
give under the previous paragraph, plus a right to possession of the
|
||||
Corresponding Source of the work from the predecessor in interest, if
|
||||
the predecessor has it or can get it with reasonable efforts.
|
||||
|
||||
You may not impose any further restrictions on the exercise of the
|
||||
rights granted or affirmed under this License. For example, you may
|
||||
not impose a license fee, royalty, or other charge for exercise of
|
||||
rights granted under this License, and you may not initiate litigation
|
||||
(including a cross-claim or counterclaim in a lawsuit) alleging that
|
||||
any patent claim is infringed by making, using, selling, offering for
|
||||
sale, or importing the Program or any portion of it.
|
||||
|
||||
11. Patents.
|
||||
|
||||
A "contributor" is a copyright holder who authorizes use under this
|
||||
License of the Program or a work on which the Program is based. The
|
||||
work thus licensed is called the contributor's "contributor version".
|
||||
|
||||
A contributor's "essential patent claims" are all patent claims
|
||||
owned or controlled by the contributor, whether already acquired or
|
||||
hereafter acquired, that would be infringed by some manner, permitted
|
||||
by this License, of making, using, or selling its contributor version,
|
||||
but do not include claims that would be infringed only as a
|
||||
consequence of further modification of the contributor version. For
|
||||
purposes of this definition, "control" includes the right to grant
|
||||
patent sublicenses in a manner consistent with the requirements of
|
||||
this License.
|
||||
|
||||
Each contributor grants you a non-exclusive, worldwide, royalty-free
|
||||
patent license under the contributor's essential patent claims, to
|
||||
make, use, sell, offer for sale, import and otherwise run, modify and
|
||||
propagate the contents of its contributor version.
|
||||
|
||||
In the following three paragraphs, a "patent license" is any express
|
||||
agreement or commitment, however denominated, not to enforce a patent
|
||||
(such as an express permission to practice a patent or covenant not to
|
||||
sue for patent infringement). To "grant" such a patent license to a
|
||||
party means to make such an agreement or commitment not to enforce a
|
||||
patent against the party.
|
||||
|
||||
If you convey a covered work, knowingly relying on a patent license,
|
||||
and the Corresponding Source of the work is not available for anyone
|
||||
to copy, free of charge and under the terms of this License, through a
|
||||
publicly available network server or other readily accessible means,
|
||||
then you must either (1) cause the Corresponding Source to be so
|
||||
available, or (2) arrange to deprive yourself of the benefit of the
|
||||
patent license for this particular work, or (3) arrange, in a manner
|
||||
consistent with the requirements of this License, to extend the patent
|
||||
license to downstream recipients. "Knowingly relying" means you have
|
||||
actual knowledge that, but for the patent license, your conveying the
|
||||
covered work in a country, or your recipient's use of the covered work
|
||||
in a country, would infringe one or more identifiable patents in that
|
||||
country that you have reason to believe are valid.
|
||||
|
||||
If, pursuant to or in connection with a single transaction or
|
||||
arrangement, you convey, or propagate by procuring conveyance of, a
|
||||
covered work, and grant a patent license to some of the parties
|
||||
receiving the covered work authorizing them to use, propagate, modify
|
||||
or convey a specific copy of the covered work, then the patent license
|
||||
you grant is automatically extended to all recipients of the covered
|
||||
work and works based on it.
|
||||
|
||||
A patent license is "discriminatory" if it does not include within
|
||||
the scope of its coverage, prohibits the exercise of, or is
|
||||
conditioned on the non-exercise of one or more of the rights that are
|
||||
specifically granted under this License. You may not convey a covered
|
||||
work if you are a party to an arrangement with a third party that is
|
||||
in the business of distributing software, under which you make payment
|
||||
to the third party based on the extent of your activity of conveying
|
||||
the work, and under which the third party grants, to any of the
|
||||
parties who would receive the covered work from you, a discriminatory
|
||||
patent license (a) in connection with copies of the covered work
|
||||
conveyed by you (or copies made from those copies), or (b) primarily
|
||||
for and in connection with specific products or compilations that
|
||||
contain the covered work, unless you entered into that arrangement,
|
||||
or that patent license was granted, prior to 28 March 2007.
|
||||
|
||||
Nothing in this License shall be construed as excluding or limiting
|
||||
any implied license or other defenses to infringement that may
|
||||
otherwise be available to you under applicable patent law.
|
||||
|
||||
12. No Surrender of Others' Freedom.
|
||||
|
||||
If conditions are imposed on you (whether by court order, agreement or
|
||||
otherwise) that contradict the conditions of this License, they do not
|
||||
excuse you from the conditions of this License. If you cannot convey a
|
||||
covered work so as to satisfy simultaneously your obligations under this
|
||||
License and any other pertinent obligations, then as a consequence you may
|
||||
not convey it at all. For example, if you agree to terms that obligate you
|
||||
to collect a royalty for further conveying from those to whom you convey
|
||||
the Program, the only way you could satisfy both those terms and this
|
||||
License would be to refrain entirely from conveying the Program.
|
||||
|
||||
13. Remote Network Interaction; Use with the GNU General Public License.
|
||||
|
||||
Notwithstanding any other provision of this License, if you modify the
|
||||
Program, your modified version must prominently offer all users
|
||||
interacting with it remotely through a computer network (if your version
|
||||
supports such interaction) an opportunity to receive the Corresponding
|
||||
Source of your version by providing access to the Corresponding Source
|
||||
from a network server at no charge, through some standard or customary
|
||||
means of facilitating copying of software. This Corresponding Source
|
||||
shall include the Corresponding Source for any work covered by version 3
|
||||
of the GNU General Public License that is incorporated pursuant to the
|
||||
following paragraph.
|
||||
|
||||
Notwithstanding any other provision of this License, you have
|
||||
permission to link or combine any covered work with a work licensed
|
||||
under version 3 of the GNU General Public License into a single
|
||||
combined work, and to convey the resulting work. The terms of this
|
||||
License will continue to apply to the part which is the covered work,
|
||||
but the work with which it is combined will remain governed by version
|
||||
3 of the GNU General Public License.
|
||||
|
||||
14. Revised Versions of this License.
|
||||
|
||||
The Free Software Foundation may publish revised and/or new versions of
|
||||
the GNU Affero General Public License from time to time. Such new versions
|
||||
will be similar in spirit to the present version, but may differ in detail to
|
||||
address new problems or concerns.
|
||||
|
||||
Each version is given a distinguishing version number. If the
|
||||
Program specifies that a certain numbered version of the GNU Affero General
|
||||
Public License "or any later version" applies to it, you have the
|
||||
option of following the terms and conditions either of that numbered
|
||||
version or of any later version published by the Free Software
|
||||
Foundation. If the Program does not specify a version number of the
|
||||
GNU Affero General Public License, you may choose any version ever published
|
||||
by the Free Software Foundation.
|
||||
|
||||
If the Program specifies that a proxy can decide which future
|
||||
versions of the GNU Affero General Public License can be used, that proxy's
|
||||
public statement of acceptance of a version permanently authorizes you
|
||||
to choose that version for the Program.
|
||||
|
||||
Later license versions may give you additional or different
|
||||
permissions. However, no additional obligations are imposed on any
|
||||
author or copyright holder as a result of your choosing to follow a
|
||||
later version.
|
||||
|
||||
15. Disclaimer of Warranty.
|
||||
|
||||
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
|
||||
APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
|
||||
HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
|
||||
OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
|
||||
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
||||
PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
|
||||
IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
|
||||
ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
|
||||
|
||||
16. Limitation of Liability.
|
||||
|
||||
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
|
||||
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
|
||||
THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
|
||||
GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
|
||||
USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
|
||||
DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
|
||||
PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
|
||||
EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
|
||||
SUCH DAMAGES.
|
||||
|
||||
17. Interpretation of Sections 15 and 16.
|
||||
|
||||
If the disclaimer of warranty and limitation of liability provided
|
||||
above cannot be given local legal effect according to their terms,
|
||||
reviewing courts shall apply local law that most closely approximates
|
||||
an absolute waiver of all civil liability in connection with the
|
||||
Program, unless a warranty or assumption of liability accompanies a
|
||||
copy of the Program in return for a fee.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
How to Apply These Terms to Your New Programs
|
||||
|
||||
If you develop a new program, and you want it to be of the greatest
|
||||
possible use to the public, the best way to achieve this is to make it
|
||||
free software which everyone can redistribute and change under these terms.
|
||||
|
||||
To do so, attach the following notices to the program. It is safest
|
||||
to attach them to the start of each source file to most effectively
|
||||
state the exclusion of warranty; and each file should have at least
|
||||
the "copyright" line and a pointer to where the full notice is found.
|
||||
|
||||
<one line to give the program's name and a brief idea of what it does.>
|
||||
Copyright (C) <year> <name of author>
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
Also add information on how to contact you by electronic and paper mail.
|
||||
|
||||
If your software can interact with users remotely through a computer
|
||||
network, you should also make sure that it provides a way for users to
|
||||
get its source. For example, if your program is a web application, its
|
||||
interface could display a "Source" link that leads users to an archive
|
||||
of the code. There are many ways you could offer source, and different
|
||||
solutions will be better for different programs; see section 13 for the
|
||||
specific requirements.
|
||||
|
||||
You should also get your employer (if you work as a programmer) or school,
|
||||
if any, to sign a "copyright disclaimer" for the program, if necessary.
|
||||
For more information on this, and how to apply and follow the GNU AGPL, see
|
||||
<https://www.gnu.org/licenses/>.
|
||||
21
README.md
21
README.md
@@ -102,6 +102,11 @@ python run.py --mode ui # UI only (port 5100)
|
||||
| `ENCRYPTION_ENABLED` | `false` | Enable server-side encryption |
|
||||
| `KMS_ENABLED` | `false` | Enable Key Management Service |
|
||||
| `LOG_LEVEL` | `INFO` | Logging verbosity |
|
||||
| `SIGV4_TIMESTAMP_TOLERANCE_SECONDS` | `900` | Max time skew for SigV4 requests |
|
||||
| `PRESIGNED_URL_MAX_EXPIRY_SECONDS` | `604800` | Max presigned URL expiry (7 days) |
|
||||
| `REPLICATION_CONNECT_TIMEOUT_SECONDS` | `5` | Replication connection timeout |
|
||||
| `SITE_SYNC_ENABLED` | `false` | Enable bi-directional site sync |
|
||||
| `OBJECT_TAG_LIMIT` | `50` | Maximum tags per object |
|
||||
|
||||
## Data Layout
|
||||
|
||||
@@ -149,19 +154,13 @@ All endpoints require AWS Signature Version 4 authentication unless using presig
|
||||
| `POST` | `/<bucket>/<key>?uploadId=X` | Complete multipart upload |
|
||||
| `DELETE` | `/<bucket>/<key>?uploadId=X` | Abort multipart upload |
|
||||
|
||||
### Presigned URLs
|
||||
### Bucket Policies (S3-compatible)
|
||||
|
||||
| Method | Endpoint | Description |
|
||||
|--------|----------|-------------|
|
||||
| `POST` | `/presign/<bucket>/<key>` | Generate presigned URL |
|
||||
|
||||
### Bucket Policies
|
||||
|
||||
| Method | Endpoint | Description |
|
||||
|--------|----------|-------------|
|
||||
| `GET` | `/bucket-policy/<bucket>` | Get bucket policy |
|
||||
| `PUT` | `/bucket-policy/<bucket>` | Set bucket policy |
|
||||
| `DELETE` | `/bucket-policy/<bucket>` | Delete bucket policy |
|
||||
| `GET` | `/<bucket>?policy` | Get bucket policy |
|
||||
| `PUT` | `/<bucket>?policy` | Set bucket policy |
|
||||
| `DELETE` | `/<bucket>?policy` | Delete bucket policy |
|
||||
|
||||
### Versioning
|
||||
|
||||
@@ -175,7 +174,7 @@ All endpoints require AWS Signature Version 4 authentication unless using presig
|
||||
|
||||
| Method | Endpoint | Description |
|
||||
|--------|----------|-------------|
|
||||
| `GET` | `/healthz` | Health check endpoint |
|
||||
| `GET` | `/myfsio/health` | Health check endpoint |
|
||||
|
||||
## IAM & Access Control
|
||||
|
||||
|
||||
327
app/__init__.py
327
app/__init__.py
@@ -1,6 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import html as html_module
|
||||
import logging
|
||||
import mimetypes
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import time
|
||||
@@ -10,12 +13,14 @@ from pathlib import Path
|
||||
from datetime import timedelta
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from flask import Flask, g, has_request_context, redirect, render_template, request, url_for
|
||||
from flask import Flask, Response, g, has_request_context, redirect, render_template, request, url_for
|
||||
from flask_cors import CORS
|
||||
from flask_wtf.csrf import CSRFError
|
||||
from werkzeug.middleware.proxy_fix import ProxyFix
|
||||
|
||||
from .access_logging import AccessLoggingService
|
||||
from .operation_metrics import OperationMetricsCollector, classify_endpoint
|
||||
from .compression import GzipMiddleware
|
||||
from .acl import AclService
|
||||
from .bucket_policies import BucketPolicyStore
|
||||
from .config import AppConfig
|
||||
@@ -29,8 +34,10 @@ from .notifications import NotificationService
|
||||
from .object_lock import ObjectLockService
|
||||
from .replication import ReplicationManager
|
||||
from .secret_store import EphemeralSecretStore
|
||||
from .storage import ObjectStorage
|
||||
from .site_registry import SiteRegistry, SiteInfo
|
||||
from .storage import ObjectStorage, StorageError
|
||||
from .version import get_version
|
||||
from .website_domains import WebsiteDomainStore
|
||||
|
||||
|
||||
def _migrate_config_file(active_path: Path, legacy_paths: List[Path]) -> Path:
|
||||
@@ -87,7 +94,18 @@ def create_app(
|
||||
app.config.setdefault("WTF_CSRF_ENABLED", False)
|
||||
|
||||
# Trust X-Forwarded-* headers from proxies
|
||||
app.wsgi_app = ProxyFix(app.wsgi_app, x_for=1, x_proto=1, x_host=1, x_prefix=1)
|
||||
num_proxies = app.config.get("NUM_TRUSTED_PROXIES", 1)
|
||||
if num_proxies:
|
||||
if "NUM_TRUSTED_PROXIES" not in os.environ:
|
||||
logging.getLogger(__name__).warning(
|
||||
"NUM_TRUSTED_PROXIES not set, defaulting to 1. "
|
||||
"Set NUM_TRUSTED_PROXIES=0 if not behind a reverse proxy."
|
||||
)
|
||||
app.wsgi_app = ProxyFix(app.wsgi_app, x_for=num_proxies, x_proto=num_proxies, x_host=num_proxies, x_prefix=num_proxies)
|
||||
|
||||
# Enable gzip compression for responses (10-20x smaller JSON payloads)
|
||||
if app.config.get("ENABLE_GZIP", True):
|
||||
app.wsgi_app = GzipMiddleware(app.wsgi_app, compression_level=6)
|
||||
|
||||
_configure_cors(app)
|
||||
_configure_logging(app)
|
||||
@@ -95,7 +113,17 @@ def create_app(
|
||||
limiter.init_app(app)
|
||||
csrf.init_app(app)
|
||||
|
||||
storage = ObjectStorage(Path(app.config["STORAGE_ROOT"]))
|
||||
storage = ObjectStorage(
|
||||
Path(app.config["STORAGE_ROOT"]),
|
||||
cache_ttl=app.config.get("OBJECT_CACHE_TTL", 60),
|
||||
object_cache_max_size=app.config.get("OBJECT_CACHE_MAX_SIZE", 100),
|
||||
bucket_config_cache_ttl=app.config.get("BUCKET_CONFIG_CACHE_TTL_SECONDS", 30.0),
|
||||
object_key_max_length_bytes=app.config.get("OBJECT_KEY_MAX_LENGTH_BYTES", 1024),
|
||||
)
|
||||
|
||||
if app.config.get("WARM_CACHE_ON_STARTUP", True) and not app.config.get("TESTING"):
|
||||
storage.warm_cache_async()
|
||||
|
||||
iam = IamService(
|
||||
Path(app.config["IAM_CONFIG"]),
|
||||
auth_max_attempts=app.config.get("AUTH_MAX_ATTEMPTS", 5),
|
||||
@@ -124,12 +152,33 @@ def create_app(
|
||||
)
|
||||
|
||||
connections = ConnectionStore(connections_path)
|
||||
replication = ReplicationManager(storage, connections, replication_rules_path, storage_root)
|
||||
|
||||
replication = ReplicationManager(
|
||||
storage,
|
||||
connections,
|
||||
replication_rules_path,
|
||||
storage_root,
|
||||
connect_timeout=app.config.get("REPLICATION_CONNECT_TIMEOUT_SECONDS", 5),
|
||||
read_timeout=app.config.get("REPLICATION_READ_TIMEOUT_SECONDS", 30),
|
||||
max_retries=app.config.get("REPLICATION_MAX_RETRIES", 2),
|
||||
streaming_threshold_bytes=app.config.get("REPLICATION_STREAMING_THRESHOLD_BYTES", 10 * 1024 * 1024),
|
||||
max_failures_per_bucket=app.config.get("REPLICATION_MAX_FAILURES_PER_BUCKET", 50),
|
||||
)
|
||||
|
||||
site_registry_path = config_dir / "site_registry.json"
|
||||
site_registry = SiteRegistry(site_registry_path)
|
||||
if app.config.get("SITE_ID") and not site_registry.get_local_site():
|
||||
site_registry.set_local_site(SiteInfo(
|
||||
site_id=app.config["SITE_ID"],
|
||||
endpoint=app.config.get("SITE_ENDPOINT") or "",
|
||||
region=app.config.get("SITE_REGION", "us-east-1"),
|
||||
priority=app.config.get("SITE_PRIORITY", 100),
|
||||
))
|
||||
|
||||
encryption_config = {
|
||||
"encryption_enabled": app.config.get("ENCRYPTION_ENABLED", False),
|
||||
"encryption_master_key_path": app.config.get("ENCRYPTION_MASTER_KEY_PATH"),
|
||||
"default_encryption_algorithm": app.config.get("DEFAULT_ENCRYPTION_ALGORITHM", "AES256"),
|
||||
"encryption_chunk_size_bytes": app.config.get("ENCRYPTION_CHUNK_SIZE_BYTES", 64 * 1024),
|
||||
}
|
||||
encryption_manager = EncryptionManager(encryption_config)
|
||||
|
||||
@@ -137,7 +186,12 @@ def create_app(
|
||||
if app.config.get("KMS_ENABLED", False):
|
||||
kms_keys_path = Path(app.config.get("KMS_KEYS_PATH", ""))
|
||||
kms_master_key_path = Path(app.config.get("ENCRYPTION_MASTER_KEY_PATH", ""))
|
||||
kms_manager = KMSManager(kms_keys_path, kms_master_key_path)
|
||||
kms_manager = KMSManager(
|
||||
kms_keys_path,
|
||||
kms_master_key_path,
|
||||
generate_data_key_min_bytes=app.config.get("KMS_GENERATE_DATA_KEY_MIN_BYTES", 1),
|
||||
generate_data_key_max_bytes=app.config.get("KMS_GENERATE_DATA_KEY_MAX_BYTES", 1024),
|
||||
)
|
||||
encryption_manager.set_kms_provider(kms_manager)
|
||||
|
||||
if app.config.get("ENCRYPTION_ENABLED", False):
|
||||
@@ -146,7 +200,10 @@ def create_app(
|
||||
|
||||
acl_service = AclService(storage_root)
|
||||
object_lock_service = ObjectLockService(storage_root)
|
||||
notification_service = NotificationService(storage_root)
|
||||
notification_service = NotificationService(
|
||||
storage_root,
|
||||
allow_internal_endpoints=app.config.get("ALLOW_INTERNAL_ENDPOINTS", False),
|
||||
)
|
||||
access_logging_service = AccessLoggingService(storage_root)
|
||||
access_logging_service.set_storage(storage)
|
||||
|
||||
@@ -157,6 +214,7 @@ def create_app(
|
||||
base_storage,
|
||||
interval_seconds=app.config.get("LIFECYCLE_INTERVAL_SECONDS", 3600),
|
||||
storage_root=storage_root,
|
||||
max_history_per_bucket=app.config.get("LIFECYCLE_MAX_HISTORY_PER_BUCKET", 50),
|
||||
)
|
||||
lifecycle_manager.start()
|
||||
|
||||
@@ -174,14 +232,92 @@ def create_app(
|
||||
app.extensions["object_lock"] = object_lock_service
|
||||
app.extensions["notifications"] = notification_service
|
||||
app.extensions["access_logging"] = access_logging_service
|
||||
app.extensions["site_registry"] = site_registry
|
||||
|
||||
website_domains_store = None
|
||||
if app.config.get("WEBSITE_HOSTING_ENABLED", False):
|
||||
website_domains_path = config_dir / "website_domains.json"
|
||||
website_domains_store = WebsiteDomainStore(website_domains_path)
|
||||
app.extensions["website_domains"] = website_domains_store
|
||||
|
||||
from .s3_client import S3ProxyClient
|
||||
api_base = app.config.get("API_BASE_URL") or "http://127.0.0.1:5000"
|
||||
app.extensions["s3_proxy"] = S3ProxyClient(
|
||||
api_base_url=api_base,
|
||||
region=app.config.get("AWS_REGION", "us-east-1"),
|
||||
)
|
||||
|
||||
operation_metrics_collector = None
|
||||
if app.config.get("OPERATION_METRICS_ENABLED", False):
|
||||
operation_metrics_collector = OperationMetricsCollector(
|
||||
storage_root,
|
||||
interval_minutes=app.config.get("OPERATION_METRICS_INTERVAL_MINUTES", 5),
|
||||
retention_hours=app.config.get("OPERATION_METRICS_RETENTION_HOURS", 24),
|
||||
)
|
||||
app.extensions["operation_metrics"] = operation_metrics_collector
|
||||
|
||||
system_metrics_collector = None
|
||||
if app.config.get("METRICS_HISTORY_ENABLED", False):
|
||||
from .system_metrics import SystemMetricsCollector
|
||||
system_metrics_collector = SystemMetricsCollector(
|
||||
storage_root,
|
||||
interval_minutes=app.config.get("METRICS_HISTORY_INTERVAL_MINUTES", 5),
|
||||
retention_hours=app.config.get("METRICS_HISTORY_RETENTION_HOURS", 24),
|
||||
)
|
||||
system_metrics_collector.set_storage(storage)
|
||||
app.extensions["system_metrics"] = system_metrics_collector
|
||||
|
||||
site_sync_worker = None
|
||||
if app.config.get("SITE_SYNC_ENABLED", False):
|
||||
from .site_sync import SiteSyncWorker
|
||||
site_sync_worker = SiteSyncWorker(
|
||||
storage=storage,
|
||||
connections=connections,
|
||||
replication_manager=replication,
|
||||
storage_root=storage_root,
|
||||
interval_seconds=app.config.get("SITE_SYNC_INTERVAL_SECONDS", 60),
|
||||
batch_size=app.config.get("SITE_SYNC_BATCH_SIZE", 100),
|
||||
connect_timeout=app.config.get("SITE_SYNC_CONNECT_TIMEOUT_SECONDS", 10),
|
||||
read_timeout=app.config.get("SITE_SYNC_READ_TIMEOUT_SECONDS", 120),
|
||||
max_retries=app.config.get("SITE_SYNC_MAX_RETRIES", 2),
|
||||
clock_skew_tolerance_seconds=app.config.get("SITE_SYNC_CLOCK_SKEW_TOLERANCE_SECONDS", 1.0),
|
||||
)
|
||||
site_sync_worker.start()
|
||||
app.extensions["site_sync"] = site_sync_worker
|
||||
|
||||
@app.errorhandler(500)
|
||||
def internal_error(error):
|
||||
return render_template('500.html'), 500
|
||||
wants_html = request.accept_mimetypes.accept_html
|
||||
path = request.path or ""
|
||||
if include_ui and wants_html and (path.startswith("/ui") or path == "/"):
|
||||
return render_template('500.html'), 500
|
||||
error_xml = (
|
||||
'<?xml version="1.0" encoding="UTF-8"?>'
|
||||
'<Error>'
|
||||
'<Code>InternalError</Code>'
|
||||
'<Message>An internal server error occurred</Message>'
|
||||
f'<Resource>{path}</Resource>'
|
||||
f'<RequestId>{getattr(g, "request_id", "-")}</RequestId>'
|
||||
'</Error>'
|
||||
)
|
||||
return error_xml, 500, {'Content-Type': 'application/xml'}
|
||||
|
||||
@app.errorhandler(CSRFError)
|
||||
def handle_csrf_error(e):
|
||||
return render_template('csrf_error.html', reason=e.description), 400
|
||||
wants_html = request.accept_mimetypes.accept_html
|
||||
path = request.path or ""
|
||||
if include_ui and wants_html and (path.startswith("/ui") or path == "/"):
|
||||
return render_template('csrf_error.html', reason=e.description), 400
|
||||
error_xml = (
|
||||
'<?xml version="1.0" encoding="UTF-8"?>'
|
||||
'<Error>'
|
||||
'<Code>CSRFError</Code>'
|
||||
f'<Message>{e.description}</Message>'
|
||||
f'<Resource>{path}</Resource>'
|
||||
f'<RequestId>{getattr(g, "request_id", "-")}</RequestId>'
|
||||
'</Error>'
|
||||
)
|
||||
return error_xml, 400, {'Content-Type': 'application/xml'}
|
||||
|
||||
@app.template_filter("filesizeformat")
|
||||
def filesizeformat(value: int) -> str:
|
||||
@@ -215,14 +351,41 @@ def create_app(
|
||||
except (ValueError, OSError):
|
||||
return "Unknown"
|
||||
|
||||
@app.template_filter("format_datetime")
|
||||
def format_datetime_filter(dt, include_tz: bool = True) -> str:
|
||||
"""Format datetime object as human-readable string in configured timezone."""
|
||||
from datetime import datetime, timezone as dt_timezone
|
||||
from zoneinfo import ZoneInfo
|
||||
if not dt:
|
||||
return ""
|
||||
try:
|
||||
display_tz = app.config.get("DISPLAY_TIMEZONE", "UTC")
|
||||
if display_tz and display_tz != "UTC":
|
||||
try:
|
||||
tz = ZoneInfo(display_tz)
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=dt_timezone.utc)
|
||||
dt = dt.astimezone(tz)
|
||||
except (KeyError, ValueError):
|
||||
pass
|
||||
tz_abbr = dt.strftime("%Z") or "UTC"
|
||||
if include_tz:
|
||||
return f"{dt.strftime('%b %d, %Y %H:%M')} ({tz_abbr})"
|
||||
return dt.strftime("%b %d, %Y %H:%M")
|
||||
except (ValueError, AttributeError):
|
||||
return str(dt)
|
||||
|
||||
if include_api:
|
||||
from .s3_api import s3_api_bp
|
||||
from .kms_api import kms_api_bp
|
||||
from .admin_api import admin_api_bp
|
||||
|
||||
app.register_blueprint(s3_api_bp)
|
||||
app.register_blueprint(kms_api_bp)
|
||||
app.register_blueprint(admin_api_bp)
|
||||
csrf.exempt(s3_api_bp)
|
||||
csrf.exempt(kms_api_bp)
|
||||
csrf.exempt(admin_api_bp)
|
||||
|
||||
if include_ui:
|
||||
from .ui import ui_bp
|
||||
@@ -242,9 +405,9 @@ def create_app(
|
||||
return render_template("404.html"), 404
|
||||
return error
|
||||
|
||||
@app.get("/healthz")
|
||||
@app.get("/myfsio/health")
|
||||
def healthcheck() -> Dict[str, str]:
|
||||
return {"status": "ok", "version": app.config.get("APP_VERSION", "unknown")}
|
||||
return {"status": "ok"}
|
||||
|
||||
return app
|
||||
|
||||
@@ -320,11 +483,134 @@ def _configure_logging(app: Flask) -> None:
|
||||
def _log_request_start() -> None:
|
||||
g.request_id = uuid.uuid4().hex
|
||||
g.request_started_at = time.perf_counter()
|
||||
g.request_bytes_in = request.content_length or 0
|
||||
app.logger.info(
|
||||
"Request started",
|
||||
extra={"path": request.path, "method": request.method, "remote_addr": request.remote_addr},
|
||||
)
|
||||
|
||||
@app.before_request
|
||||
def _maybe_serve_website():
|
||||
if not app.config.get("WEBSITE_HOSTING_ENABLED"):
|
||||
return None
|
||||
if request.method not in {"GET", "HEAD"}:
|
||||
return None
|
||||
host = request.host
|
||||
if ":" in host:
|
||||
host = host.rsplit(":", 1)[0]
|
||||
host = host.lower()
|
||||
store = app.extensions.get("website_domains")
|
||||
if not store:
|
||||
return None
|
||||
bucket = store.get_bucket(host)
|
||||
if not bucket:
|
||||
return None
|
||||
storage = app.extensions["object_storage"]
|
||||
if not storage.bucket_exists(bucket):
|
||||
return _website_error_response(404, "Not Found")
|
||||
website_config = storage.get_bucket_website(bucket)
|
||||
if not website_config:
|
||||
return _website_error_response(404, "Not Found")
|
||||
index_doc = website_config.get("index_document", "index.html")
|
||||
error_doc = website_config.get("error_document")
|
||||
req_path = request.path.lstrip("/")
|
||||
if not req_path or req_path.endswith("/"):
|
||||
object_key = req_path + index_doc
|
||||
else:
|
||||
object_key = req_path
|
||||
try:
|
||||
obj_path = storage.get_object_path(bucket, object_key)
|
||||
except (StorageError, OSError):
|
||||
if object_key == req_path:
|
||||
try:
|
||||
obj_path = storage.get_object_path(bucket, req_path + "/" + index_doc)
|
||||
object_key = req_path + "/" + index_doc
|
||||
except (StorageError, OSError):
|
||||
return _serve_website_error(storage, bucket, error_doc, 404)
|
||||
else:
|
||||
return _serve_website_error(storage, bucket, error_doc, 404)
|
||||
content_type = mimetypes.guess_type(object_key)[0] or "application/octet-stream"
|
||||
is_encrypted = False
|
||||
try:
|
||||
metadata = storage.get_object_metadata(bucket, object_key)
|
||||
is_encrypted = "x-amz-server-side-encryption" in metadata
|
||||
except (StorageError, OSError):
|
||||
pass
|
||||
if request.method == "HEAD":
|
||||
response = Response(status=200)
|
||||
if is_encrypted and hasattr(storage, "get_object_data"):
|
||||
try:
|
||||
data, _ = storage.get_object_data(bucket, object_key)
|
||||
response.headers["Content-Length"] = len(data)
|
||||
except (StorageError, OSError):
|
||||
return _website_error_response(500, "Internal Server Error")
|
||||
else:
|
||||
try:
|
||||
stat = obj_path.stat()
|
||||
response.headers["Content-Length"] = stat.st_size
|
||||
except OSError:
|
||||
return _website_error_response(500, "Internal Server Error")
|
||||
response.headers["Content-Type"] = content_type
|
||||
return response
|
||||
if is_encrypted and hasattr(storage, "get_object_data"):
|
||||
try:
|
||||
data, _ = storage.get_object_data(bucket, object_key)
|
||||
response = Response(data, mimetype=content_type)
|
||||
response.headers["Content-Length"] = len(data)
|
||||
return response
|
||||
except (StorageError, OSError):
|
||||
return _website_error_response(500, "Internal Server Error")
|
||||
def _stream(file_path):
|
||||
with file_path.open("rb") as f:
|
||||
while True:
|
||||
chunk = f.read(65536)
|
||||
if not chunk:
|
||||
break
|
||||
yield chunk
|
||||
try:
|
||||
stat = obj_path.stat()
|
||||
response = Response(_stream(obj_path), mimetype=content_type, direct_passthrough=True)
|
||||
response.headers["Content-Length"] = stat.st_size
|
||||
return response
|
||||
except OSError:
|
||||
return _website_error_response(500, "Internal Server Error")
|
||||
|
||||
def _serve_website_error(storage, bucket, error_doc_key, status_code):
|
||||
if not error_doc_key:
|
||||
return _website_error_response(status_code, "Not Found" if status_code == 404 else "Error")
|
||||
try:
|
||||
obj_path = storage.get_object_path(bucket, error_doc_key)
|
||||
except (StorageError, OSError):
|
||||
return _website_error_response(status_code, "Not Found")
|
||||
content_type = mimetypes.guess_type(error_doc_key)[0] or "text/html"
|
||||
is_encrypted = False
|
||||
try:
|
||||
metadata = storage.get_object_metadata(bucket, error_doc_key)
|
||||
is_encrypted = "x-amz-server-side-encryption" in metadata
|
||||
except (StorageError, OSError):
|
||||
pass
|
||||
if is_encrypted and hasattr(storage, "get_object_data"):
|
||||
try:
|
||||
data, _ = storage.get_object_data(bucket, error_doc_key)
|
||||
response = Response(data, status=status_code, mimetype=content_type)
|
||||
response.headers["Content-Length"] = len(data)
|
||||
return response
|
||||
except (StorageError, OSError):
|
||||
return _website_error_response(status_code, "Not Found")
|
||||
try:
|
||||
data = obj_path.read_bytes()
|
||||
response = Response(data, status=status_code, mimetype=content_type)
|
||||
response.headers["Content-Length"] = len(data)
|
||||
return response
|
||||
except OSError:
|
||||
return _website_error_response(status_code, "Not Found")
|
||||
|
||||
def _website_error_response(status_code, message):
|
||||
safe_msg = html_module.escape(str(message))
|
||||
safe_code = html_module.escape(str(status_code))
|
||||
body = f"<html><head><title>{safe_code} {safe_msg}</title></head><body><h1>{safe_code} {safe_msg}</h1></body></html>"
|
||||
return Response(body, status=status_code, mimetype="text/html")
|
||||
|
||||
@app.after_request
|
||||
def _log_request_end(response):
|
||||
duration_ms = 0.0
|
||||
@@ -341,4 +627,21 @@ def _configure_logging(app: Flask) -> None:
|
||||
},
|
||||
)
|
||||
response.headers["X-Request-Duration-ms"] = f"{duration_ms:.2f}"
|
||||
|
||||
operation_metrics = app.extensions.get("operation_metrics")
|
||||
if operation_metrics:
|
||||
bytes_in = getattr(g, "request_bytes_in", 0)
|
||||
bytes_out = response.content_length or 0
|
||||
error_code = getattr(g, "s3_error_code", None)
|
||||
endpoint_type = classify_endpoint(request.path)
|
||||
operation_metrics.record_request(
|
||||
method=request.method,
|
||||
endpoint_type=endpoint_type,
|
||||
status_code=response.status_code,
|
||||
latency_ms=duration_ms,
|
||||
bytes_in=bytes_in,
|
||||
bytes_out=bytes_out,
|
||||
error_code=error_code,
|
||||
)
|
||||
|
||||
return response
|
||||
|
||||
778
app/admin_api.py
Normal file
778
app/admin_api.py
Normal file
@@ -0,0 +1,778 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import ipaddress
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import socket
|
||||
import time
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import requests
|
||||
from flask import Blueprint, Response, current_app, jsonify, request
|
||||
|
||||
from .connections import ConnectionStore
|
||||
from .extensions import limiter
|
||||
from .iam import IamError, Principal
|
||||
from .replication import ReplicationManager
|
||||
from .site_registry import PeerSite, SiteInfo, SiteRegistry
|
||||
from .website_domains import WebsiteDomainStore, normalize_domain, is_valid_domain
|
||||
|
||||
|
||||
def _is_safe_url(url: str, allow_internal: bool = False) -> bool:
|
||||
"""Check if a URL is safe to make requests to (not internal/private).
|
||||
|
||||
Args:
|
||||
url: The URL to check.
|
||||
allow_internal: If True, allows internal/private IP addresses.
|
||||
Use for self-hosted deployments on internal networks.
|
||||
"""
|
||||
try:
|
||||
parsed = urlparse(url)
|
||||
hostname = parsed.hostname
|
||||
if not hostname:
|
||||
return False
|
||||
cloud_metadata_hosts = {
|
||||
"metadata.google.internal",
|
||||
"169.254.169.254",
|
||||
}
|
||||
if hostname.lower() in cloud_metadata_hosts:
|
||||
return False
|
||||
if allow_internal:
|
||||
return True
|
||||
blocked_hosts = {
|
||||
"localhost",
|
||||
"127.0.0.1",
|
||||
"0.0.0.0",
|
||||
"::1",
|
||||
"[::1]",
|
||||
}
|
||||
if hostname.lower() in blocked_hosts:
|
||||
return False
|
||||
try:
|
||||
resolved_ip = socket.gethostbyname(hostname)
|
||||
ip = ipaddress.ip_address(resolved_ip)
|
||||
if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved:
|
||||
return False
|
||||
except (socket.gaierror, ValueError):
|
||||
return False
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _validate_endpoint(endpoint: str) -> Optional[str]:
|
||||
"""Validate endpoint URL format. Returns error message or None."""
|
||||
try:
|
||||
parsed = urlparse(endpoint)
|
||||
if not parsed.scheme or parsed.scheme not in ("http", "https"):
|
||||
return "Endpoint must be http or https URL"
|
||||
if not parsed.netloc:
|
||||
return "Endpoint must have a host"
|
||||
return None
|
||||
except Exception:
|
||||
return "Invalid endpoint URL"
|
||||
|
||||
|
||||
def _validate_priority(priority: Any) -> Optional[str]:
|
||||
"""Validate priority value. Returns error message or None."""
|
||||
try:
|
||||
p = int(priority)
|
||||
if p < 0 or p > 1000:
|
||||
return "Priority must be between 0 and 1000"
|
||||
return None
|
||||
except (TypeError, ValueError):
|
||||
return "Priority must be an integer"
|
||||
|
||||
|
||||
def _validate_region(region: str) -> Optional[str]:
|
||||
"""Validate region format. Returns error message or None."""
|
||||
if not re.match(r"^[a-z]{2,}-[a-z]+-\d+$", region):
|
||||
return "Region must match format like us-east-1"
|
||||
return None
|
||||
|
||||
|
||||
def _validate_site_id(site_id: str) -> Optional[str]:
|
||||
"""Validate site_id format. Returns error message or None."""
|
||||
if not site_id or len(site_id) > 63:
|
||||
return "site_id must be 1-63 characters"
|
||||
if not re.match(r'^[a-zA-Z0-9][a-zA-Z0-9_-]*$', site_id):
|
||||
return "site_id must start with alphanumeric and contain only alphanumeric, hyphens, underscores"
|
||||
return None
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
admin_api_bp = Blueprint("admin_api", __name__, url_prefix="/admin")
|
||||
|
||||
|
||||
def _require_principal() -> Tuple[Optional[Principal], Optional[Tuple[Dict[str, Any], int]]]:
|
||||
from .s3_api import _require_principal as s3_require_principal
|
||||
return s3_require_principal()
|
||||
|
||||
|
||||
def _require_admin() -> Tuple[Optional[Principal], Optional[Tuple[Dict[str, Any], int]]]:
|
||||
principal, error = _require_principal()
|
||||
if error:
|
||||
return None, error
|
||||
|
||||
try:
|
||||
_iam().authorize(principal, None, "iam:*")
|
||||
return principal, None
|
||||
except IamError:
|
||||
return None, _json_error("AccessDenied", "Admin access required", 403)
|
||||
|
||||
|
||||
def _site_registry() -> SiteRegistry:
|
||||
return current_app.extensions["site_registry"]
|
||||
|
||||
|
||||
def _connections() -> ConnectionStore:
|
||||
return current_app.extensions["connections"]
|
||||
|
||||
|
||||
def _replication() -> ReplicationManager:
|
||||
return current_app.extensions["replication"]
|
||||
|
||||
|
||||
def _iam():
|
||||
return current_app.extensions["iam"]
|
||||
|
||||
|
||||
def _json_error(code: str, message: str, status: int) -> Tuple[Dict[str, Any], int]:
|
||||
return {"error": {"code": code, "message": message}}, status
|
||||
|
||||
|
||||
def _get_admin_rate_limit() -> str:
|
||||
return current_app.config.get("RATE_LIMIT_ADMIN", "60 per minute")
|
||||
|
||||
|
||||
@admin_api_bp.route("/site", methods=["GET"])
|
||||
@limiter.limit(lambda: _get_admin_rate_limit())
|
||||
def get_local_site():
|
||||
principal, error = _require_admin()
|
||||
if error:
|
||||
return error
|
||||
|
||||
registry = _site_registry()
|
||||
local_site = registry.get_local_site()
|
||||
|
||||
if local_site:
|
||||
return jsonify(local_site.to_dict())
|
||||
|
||||
config_site_id = current_app.config.get("SITE_ID")
|
||||
config_endpoint = current_app.config.get("SITE_ENDPOINT")
|
||||
|
||||
if config_site_id:
|
||||
return jsonify({
|
||||
"site_id": config_site_id,
|
||||
"endpoint": config_endpoint or "",
|
||||
"region": current_app.config.get("SITE_REGION", "us-east-1"),
|
||||
"priority": current_app.config.get("SITE_PRIORITY", 100),
|
||||
"display_name": config_site_id,
|
||||
"source": "environment",
|
||||
})
|
||||
|
||||
return _json_error("NotFound", "Local site not configured", 404)
|
||||
|
||||
|
||||
@admin_api_bp.route("/site", methods=["PUT"])
|
||||
@limiter.limit(lambda: _get_admin_rate_limit())
|
||||
def update_local_site():
|
||||
principal, error = _require_admin()
|
||||
if error:
|
||||
return error
|
||||
|
||||
payload = request.get_json(silent=True) or {}
|
||||
|
||||
site_id = payload.get("site_id")
|
||||
endpoint = payload.get("endpoint")
|
||||
|
||||
if not site_id:
|
||||
return _json_error("ValidationError", "site_id is required", 400)
|
||||
|
||||
site_id_error = _validate_site_id(site_id)
|
||||
if site_id_error:
|
||||
return _json_error("ValidationError", site_id_error, 400)
|
||||
|
||||
if endpoint:
|
||||
endpoint_error = _validate_endpoint(endpoint)
|
||||
if endpoint_error:
|
||||
return _json_error("ValidationError", endpoint_error, 400)
|
||||
|
||||
if "priority" in payload:
|
||||
priority_error = _validate_priority(payload["priority"])
|
||||
if priority_error:
|
||||
return _json_error("ValidationError", priority_error, 400)
|
||||
|
||||
if "region" in payload:
|
||||
region_error = _validate_region(payload["region"])
|
||||
if region_error:
|
||||
return _json_error("ValidationError", region_error, 400)
|
||||
|
||||
registry = _site_registry()
|
||||
existing = registry.get_local_site()
|
||||
|
||||
site = SiteInfo(
|
||||
site_id=site_id,
|
||||
endpoint=endpoint or "",
|
||||
region=payload.get("region", "us-east-1"),
|
||||
priority=payload.get("priority", 100),
|
||||
display_name=payload.get("display_name", site_id),
|
||||
created_at=existing.created_at if existing else None,
|
||||
)
|
||||
|
||||
registry.set_local_site(site)
|
||||
|
||||
logger.info("Local site updated", extra={"site_id": site_id, "principal": principal.access_key})
|
||||
return jsonify(site.to_dict())
|
||||
|
||||
|
||||
@admin_api_bp.route("/sites", methods=["GET"])
|
||||
@limiter.limit(lambda: _get_admin_rate_limit())
|
||||
def list_all_sites():
|
||||
principal, error = _require_admin()
|
||||
if error:
|
||||
return error
|
||||
|
||||
registry = _site_registry()
|
||||
local = registry.get_local_site()
|
||||
peers = registry.list_peers()
|
||||
|
||||
result = {
|
||||
"local": local.to_dict() if local else None,
|
||||
"peers": [peer.to_dict() for peer in peers],
|
||||
"total_peers": len(peers),
|
||||
}
|
||||
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
@admin_api_bp.route("/sites", methods=["POST"])
|
||||
@limiter.limit(lambda: _get_admin_rate_limit())
|
||||
def register_peer_site():
|
||||
principal, error = _require_admin()
|
||||
if error:
|
||||
return error
|
||||
|
||||
payload = request.get_json(silent=True) or {}
|
||||
|
||||
site_id = payload.get("site_id")
|
||||
endpoint = payload.get("endpoint")
|
||||
|
||||
if not site_id:
|
||||
return _json_error("ValidationError", "site_id is required", 400)
|
||||
|
||||
site_id_error = _validate_site_id(site_id)
|
||||
if site_id_error:
|
||||
return _json_error("ValidationError", site_id_error, 400)
|
||||
|
||||
if not endpoint:
|
||||
return _json_error("ValidationError", "endpoint is required", 400)
|
||||
|
||||
endpoint_error = _validate_endpoint(endpoint)
|
||||
if endpoint_error:
|
||||
return _json_error("ValidationError", endpoint_error, 400)
|
||||
|
||||
region = payload.get("region", "us-east-1")
|
||||
region_error = _validate_region(region)
|
||||
if region_error:
|
||||
return _json_error("ValidationError", region_error, 400)
|
||||
|
||||
priority = payload.get("priority", 100)
|
||||
priority_error = _validate_priority(priority)
|
||||
if priority_error:
|
||||
return _json_error("ValidationError", priority_error, 400)
|
||||
|
||||
registry = _site_registry()
|
||||
|
||||
if registry.get_peer(site_id):
|
||||
return _json_error("AlreadyExists", f"Peer site '{site_id}' already exists", 409)
|
||||
|
||||
connection_id = payload.get("connection_id")
|
||||
if connection_id:
|
||||
if not _connections().get(connection_id):
|
||||
return _json_error("ValidationError", f"Connection '{connection_id}' not found", 400)
|
||||
|
||||
peer = PeerSite(
|
||||
site_id=site_id,
|
||||
endpoint=endpoint,
|
||||
region=region,
|
||||
priority=int(priority),
|
||||
display_name=payload.get("display_name", site_id),
|
||||
connection_id=connection_id,
|
||||
)
|
||||
|
||||
registry.add_peer(peer)
|
||||
|
||||
logger.info("Peer site registered", extra={"site_id": site_id, "principal": principal.access_key})
|
||||
return jsonify(peer.to_dict()), 201
|
||||
|
||||
|
||||
@admin_api_bp.route("/sites/<site_id>", methods=["GET"])
|
||||
@limiter.limit(lambda: _get_admin_rate_limit())
|
||||
def get_peer_site(site_id: str):
|
||||
principal, error = _require_admin()
|
||||
if error:
|
||||
return error
|
||||
|
||||
registry = _site_registry()
|
||||
peer = registry.get_peer(site_id)
|
||||
|
||||
if not peer:
|
||||
return _json_error("NotFound", f"Peer site '{site_id}' not found", 404)
|
||||
|
||||
return jsonify(peer.to_dict())
|
||||
|
||||
|
||||
@admin_api_bp.route("/sites/<site_id>", methods=["PUT"])
|
||||
@limiter.limit(lambda: _get_admin_rate_limit())
|
||||
def update_peer_site(site_id: str):
|
||||
principal, error = _require_admin()
|
||||
if error:
|
||||
return error
|
||||
|
||||
registry = _site_registry()
|
||||
existing = registry.get_peer(site_id)
|
||||
|
||||
if not existing:
|
||||
return _json_error("NotFound", f"Peer site '{site_id}' not found", 404)
|
||||
|
||||
payload = request.get_json(silent=True) or {}
|
||||
|
||||
if "endpoint" in payload:
|
||||
endpoint_error = _validate_endpoint(payload["endpoint"])
|
||||
if endpoint_error:
|
||||
return _json_error("ValidationError", endpoint_error, 400)
|
||||
|
||||
if "priority" in payload:
|
||||
priority_error = _validate_priority(payload["priority"])
|
||||
if priority_error:
|
||||
return _json_error("ValidationError", priority_error, 400)
|
||||
|
||||
if "region" in payload:
|
||||
region_error = _validate_region(payload["region"])
|
||||
if region_error:
|
||||
return _json_error("ValidationError", region_error, 400)
|
||||
|
||||
if "connection_id" in payload:
|
||||
if payload["connection_id"] and not _connections().get(payload["connection_id"]):
|
||||
return _json_error("ValidationError", f"Connection '{payload['connection_id']}' not found", 400)
|
||||
|
||||
peer = PeerSite(
|
||||
site_id=site_id,
|
||||
endpoint=payload.get("endpoint", existing.endpoint),
|
||||
region=payload.get("region", existing.region),
|
||||
priority=payload.get("priority", existing.priority),
|
||||
display_name=payload.get("display_name", existing.display_name),
|
||||
connection_id=payload.get("connection_id", existing.connection_id),
|
||||
created_at=existing.created_at,
|
||||
is_healthy=existing.is_healthy,
|
||||
last_health_check=existing.last_health_check,
|
||||
)
|
||||
|
||||
registry.update_peer(peer)
|
||||
|
||||
logger.info("Peer site updated", extra={"site_id": site_id, "principal": principal.access_key})
|
||||
return jsonify(peer.to_dict())
|
||||
|
||||
|
||||
@admin_api_bp.route("/sites/<site_id>", methods=["DELETE"])
|
||||
@limiter.limit(lambda: _get_admin_rate_limit())
|
||||
def delete_peer_site(site_id: str):
|
||||
principal, error = _require_admin()
|
||||
if error:
|
||||
return error
|
||||
|
||||
registry = _site_registry()
|
||||
|
||||
if not registry.delete_peer(site_id):
|
||||
return _json_error("NotFound", f"Peer site '{site_id}' not found", 404)
|
||||
|
||||
logger.info("Peer site deleted", extra={"site_id": site_id, "principal": principal.access_key})
|
||||
return Response(status=204)
|
||||
|
||||
|
||||
@admin_api_bp.route("/sites/<site_id>/health", methods=["GET"])
|
||||
@limiter.limit(lambda: _get_admin_rate_limit())
|
||||
def check_peer_health(site_id: str):
|
||||
principal, error = _require_admin()
|
||||
if error:
|
||||
return error
|
||||
|
||||
registry = _site_registry()
|
||||
peer = registry.get_peer(site_id)
|
||||
|
||||
if not peer:
|
||||
return _json_error("NotFound", f"Peer site '{site_id}' not found", 404)
|
||||
|
||||
is_healthy = False
|
||||
error_message = None
|
||||
|
||||
if peer.connection_id:
|
||||
connection = _connections().get(peer.connection_id)
|
||||
if connection:
|
||||
is_healthy = _replication().check_endpoint_health(connection)
|
||||
else:
|
||||
error_message = f"Connection '{peer.connection_id}' not found"
|
||||
else:
|
||||
error_message = "No connection configured for this peer"
|
||||
|
||||
registry.update_health(site_id, is_healthy)
|
||||
|
||||
result = {
|
||||
"site_id": site_id,
|
||||
"is_healthy": is_healthy,
|
||||
"checked_at": time.time(),
|
||||
}
|
||||
if error_message:
|
||||
result["error"] = error_message
|
||||
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
@admin_api_bp.route("/topology", methods=["GET"])
|
||||
@limiter.limit(lambda: _get_admin_rate_limit())
|
||||
def get_topology():
|
||||
principal, error = _require_admin()
|
||||
if error:
|
||||
return error
|
||||
|
||||
registry = _site_registry()
|
||||
local = registry.get_local_site()
|
||||
peers = registry.list_peers()
|
||||
|
||||
sites = []
|
||||
|
||||
if local:
|
||||
sites.append({
|
||||
**local.to_dict(),
|
||||
"is_local": True,
|
||||
"is_healthy": True,
|
||||
})
|
||||
|
||||
for peer in peers:
|
||||
sites.append({
|
||||
**peer.to_dict(),
|
||||
"is_local": False,
|
||||
})
|
||||
|
||||
sites.sort(key=lambda s: s.get("priority", 100))
|
||||
|
||||
return jsonify({
|
||||
"sites": sites,
|
||||
"total": len(sites),
|
||||
"healthy_count": sum(1 for s in sites if s.get("is_healthy")),
|
||||
})
|
||||
|
||||
|
||||
@admin_api_bp.route("/sites/<site_id>/bidirectional-status", methods=["GET"])
|
||||
@limiter.limit(lambda: _get_admin_rate_limit())
|
||||
def check_bidirectional_status(site_id: str):
|
||||
principal, error = _require_admin()
|
||||
if error:
|
||||
return error
|
||||
|
||||
registry = _site_registry()
|
||||
peer = registry.get_peer(site_id)
|
||||
|
||||
if not peer:
|
||||
return _json_error("NotFound", f"Peer site '{site_id}' not found", 404)
|
||||
|
||||
local_site = registry.get_local_site()
|
||||
replication = _replication()
|
||||
local_rules = replication.list_rules()
|
||||
|
||||
local_bidir_rules = []
|
||||
for rule in local_rules:
|
||||
if rule.target_connection_id == peer.connection_id and rule.mode == "bidirectional":
|
||||
local_bidir_rules.append({
|
||||
"bucket_name": rule.bucket_name,
|
||||
"target_bucket": rule.target_bucket,
|
||||
"enabled": rule.enabled,
|
||||
})
|
||||
|
||||
result = {
|
||||
"site_id": site_id,
|
||||
"local_site_id": local_site.site_id if local_site else None,
|
||||
"local_endpoint": local_site.endpoint if local_site else None,
|
||||
"local_bidirectional_rules": local_bidir_rules,
|
||||
"local_site_sync_enabled": current_app.config.get("SITE_SYNC_ENABLED", False),
|
||||
"remote_status": None,
|
||||
"issues": [],
|
||||
"is_fully_configured": False,
|
||||
}
|
||||
|
||||
if not local_site or not local_site.site_id:
|
||||
result["issues"].append({
|
||||
"code": "NO_LOCAL_SITE_ID",
|
||||
"message": "Local site identity not configured",
|
||||
"severity": "error",
|
||||
})
|
||||
|
||||
if not local_site or not local_site.endpoint:
|
||||
result["issues"].append({
|
||||
"code": "NO_LOCAL_ENDPOINT",
|
||||
"message": "Local site endpoint not configured (remote site cannot reach back)",
|
||||
"severity": "error",
|
||||
})
|
||||
|
||||
if not peer.connection_id:
|
||||
result["issues"].append({
|
||||
"code": "NO_CONNECTION",
|
||||
"message": "No connection configured for this peer",
|
||||
"severity": "error",
|
||||
})
|
||||
return jsonify(result)
|
||||
|
||||
connection = _connections().get(peer.connection_id)
|
||||
if not connection:
|
||||
result["issues"].append({
|
||||
"code": "CONNECTION_NOT_FOUND",
|
||||
"message": f"Connection '{peer.connection_id}' not found",
|
||||
"severity": "error",
|
||||
})
|
||||
return jsonify(result)
|
||||
|
||||
if not local_bidir_rules:
|
||||
result["issues"].append({
|
||||
"code": "NO_LOCAL_BIDIRECTIONAL_RULES",
|
||||
"message": "No bidirectional replication rules configured on this site",
|
||||
"severity": "warning",
|
||||
})
|
||||
|
||||
if not result["local_site_sync_enabled"]:
|
||||
result["issues"].append({
|
||||
"code": "SITE_SYNC_DISABLED",
|
||||
"message": "Site sync worker is disabled (SITE_SYNC_ENABLED=false). Pull operations will not work.",
|
||||
"severity": "warning",
|
||||
})
|
||||
|
||||
if not replication.check_endpoint_health(connection):
|
||||
result["issues"].append({
|
||||
"code": "REMOTE_UNREACHABLE",
|
||||
"message": "Remote endpoint is not reachable",
|
||||
"severity": "error",
|
||||
})
|
||||
return jsonify(result)
|
||||
|
||||
allow_internal = current_app.config.get("ALLOW_INTERNAL_ENDPOINTS", False)
|
||||
if not _is_safe_url(peer.endpoint, allow_internal=allow_internal):
|
||||
result["issues"].append({
|
||||
"code": "ENDPOINT_NOT_ALLOWED",
|
||||
"message": "Peer endpoint points to cloud metadata service (SSRF protection)",
|
||||
"severity": "error",
|
||||
})
|
||||
return jsonify(result)
|
||||
|
||||
try:
|
||||
admin_url = peer.endpoint.rstrip("/") + "/admin/sites"
|
||||
resp = requests.get(
|
||||
admin_url,
|
||||
timeout=10,
|
||||
headers={
|
||||
"Accept": "application/json",
|
||||
"X-Access-Key": connection.access_key,
|
||||
"X-Secret-Key": connection.secret_key,
|
||||
},
|
||||
)
|
||||
|
||||
if resp.status_code == 200:
|
||||
try:
|
||||
remote_data = resp.json()
|
||||
if not isinstance(remote_data, dict):
|
||||
raise ValueError("Expected JSON object")
|
||||
remote_local = remote_data.get("local")
|
||||
if remote_local is not None and not isinstance(remote_local, dict):
|
||||
raise ValueError("Expected 'local' to be an object")
|
||||
remote_peers = remote_data.get("peers", [])
|
||||
if not isinstance(remote_peers, list):
|
||||
raise ValueError("Expected 'peers' to be a list")
|
||||
except (ValueError, json.JSONDecodeError) as e:
|
||||
logger.warning("Invalid JSON from remote admin API: %s", e)
|
||||
result["remote_status"] = {"reachable": True, "invalid_response": True}
|
||||
result["issues"].append({
|
||||
"code": "REMOTE_INVALID_RESPONSE",
|
||||
"message": "Remote admin API returned invalid JSON",
|
||||
"severity": "warning",
|
||||
})
|
||||
return jsonify(result)
|
||||
|
||||
result["remote_status"] = {
|
||||
"reachable": True,
|
||||
"local_site": remote_local,
|
||||
"site_sync_enabled": None,
|
||||
"has_peer_for_us": False,
|
||||
"peer_connection_configured": False,
|
||||
"has_bidirectional_rules_for_us": False,
|
||||
}
|
||||
|
||||
for rp in remote_peers:
|
||||
if not isinstance(rp, dict):
|
||||
continue
|
||||
if local_site and (
|
||||
rp.get("site_id") == local_site.site_id or
|
||||
rp.get("endpoint") == local_site.endpoint
|
||||
):
|
||||
result["remote_status"]["has_peer_for_us"] = True
|
||||
result["remote_status"]["peer_connection_configured"] = bool(rp.get("connection_id"))
|
||||
break
|
||||
|
||||
if not result["remote_status"]["has_peer_for_us"]:
|
||||
result["issues"].append({
|
||||
"code": "REMOTE_NO_PEER_FOR_US",
|
||||
"message": "Remote site does not have this site registered as a peer",
|
||||
"severity": "error",
|
||||
})
|
||||
elif not result["remote_status"]["peer_connection_configured"]:
|
||||
result["issues"].append({
|
||||
"code": "REMOTE_NO_CONNECTION_FOR_US",
|
||||
"message": "Remote site has us as peer but no connection configured (cannot push back)",
|
||||
"severity": "error",
|
||||
})
|
||||
elif resp.status_code == 401 or resp.status_code == 403:
|
||||
result["remote_status"] = {
|
||||
"reachable": True,
|
||||
"admin_access_denied": True,
|
||||
}
|
||||
result["issues"].append({
|
||||
"code": "REMOTE_ADMIN_ACCESS_DENIED",
|
||||
"message": "Cannot verify remote configuration (admin access denied)",
|
||||
"severity": "warning",
|
||||
})
|
||||
else:
|
||||
result["remote_status"] = {
|
||||
"reachable": True,
|
||||
"admin_api_error": resp.status_code,
|
||||
}
|
||||
result["issues"].append({
|
||||
"code": "REMOTE_ADMIN_API_ERROR",
|
||||
"message": f"Remote admin API returned status {resp.status_code}",
|
||||
"severity": "warning",
|
||||
})
|
||||
except requests.RequestException as e:
|
||||
logger.warning("Remote admin API unreachable: %s", e)
|
||||
result["remote_status"] = {
|
||||
"reachable": False,
|
||||
"error": "Connection failed",
|
||||
}
|
||||
result["issues"].append({
|
||||
"code": "REMOTE_ADMIN_UNREACHABLE",
|
||||
"message": "Could not reach remote admin API",
|
||||
"severity": "warning",
|
||||
})
|
||||
except Exception as e:
|
||||
logger.warning("Error checking remote bidirectional status: %s", e, exc_info=True)
|
||||
result["issues"].append({
|
||||
"code": "VERIFICATION_ERROR",
|
||||
"message": "Internal error during verification",
|
||||
"severity": "warning",
|
||||
})
|
||||
|
||||
error_issues = [i for i in result["issues"] if i["severity"] == "error"]
|
||||
result["is_fully_configured"] = len(error_issues) == 0 and len(local_bidir_rules) > 0
|
||||
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
def _website_domains() -> WebsiteDomainStore:
|
||||
return current_app.extensions["website_domains"]
|
||||
|
||||
|
||||
def _storage():
|
||||
return current_app.extensions["object_storage"]
|
||||
|
||||
|
||||
@admin_api_bp.route("/website-domains", methods=["GET"])
|
||||
@limiter.limit(lambda: _get_admin_rate_limit())
|
||||
def list_website_domains():
|
||||
principal, error = _require_admin()
|
||||
if error:
|
||||
return error
|
||||
if not current_app.config.get("WEBSITE_HOSTING_ENABLED", False):
|
||||
return _json_error("InvalidRequest", "Website hosting is not enabled", 400)
|
||||
return jsonify(_website_domains().list_all())
|
||||
|
||||
|
||||
@admin_api_bp.route("/website-domains", methods=["POST"])
|
||||
@limiter.limit(lambda: _get_admin_rate_limit())
|
||||
def create_website_domain():
|
||||
principal, error = _require_admin()
|
||||
if error:
|
||||
return error
|
||||
if not current_app.config.get("WEBSITE_HOSTING_ENABLED", False):
|
||||
return _json_error("InvalidRequest", "Website hosting is not enabled", 400)
|
||||
payload = request.get_json(silent=True) or {}
|
||||
domain = normalize_domain(payload.get("domain") or "")
|
||||
bucket = (payload.get("bucket") or "").strip()
|
||||
if not domain:
|
||||
return _json_error("ValidationError", "domain is required", 400)
|
||||
if not is_valid_domain(domain):
|
||||
return _json_error("ValidationError", f"Invalid domain: '{domain}'", 400)
|
||||
if not bucket:
|
||||
return _json_error("ValidationError", "bucket is required", 400)
|
||||
storage = _storage()
|
||||
if not storage.bucket_exists(bucket):
|
||||
return _json_error("NoSuchBucket", f"Bucket '{bucket}' does not exist", 404)
|
||||
store = _website_domains()
|
||||
existing = store.get_bucket(domain)
|
||||
if existing:
|
||||
return _json_error("Conflict", f"Domain '{domain}' is already mapped to bucket '{existing}'", 409)
|
||||
store.set_mapping(domain, bucket)
|
||||
logger.info("Website domain mapping created: %s -> %s", domain, bucket)
|
||||
return jsonify({"domain": domain, "bucket": bucket}), 201
|
||||
|
||||
|
||||
@admin_api_bp.route("/website-domains/<domain>", methods=["GET"])
|
||||
@limiter.limit(lambda: _get_admin_rate_limit())
|
||||
def get_website_domain(domain: str):
|
||||
principal, error = _require_admin()
|
||||
if error:
|
||||
return error
|
||||
if not current_app.config.get("WEBSITE_HOSTING_ENABLED", False):
|
||||
return _json_error("InvalidRequest", "Website hosting is not enabled", 400)
|
||||
domain = normalize_domain(domain)
|
||||
bucket = _website_domains().get_bucket(domain)
|
||||
if not bucket:
|
||||
return _json_error("NotFound", f"No mapping found for domain '{domain}'", 404)
|
||||
return jsonify({"domain": domain, "bucket": bucket})
|
||||
|
||||
|
||||
@admin_api_bp.route("/website-domains/<domain>", methods=["PUT"])
|
||||
@limiter.limit(lambda: _get_admin_rate_limit())
|
||||
def update_website_domain(domain: str):
|
||||
principal, error = _require_admin()
|
||||
if error:
|
||||
return error
|
||||
if not current_app.config.get("WEBSITE_HOSTING_ENABLED", False):
|
||||
return _json_error("InvalidRequest", "Website hosting is not enabled", 400)
|
||||
domain = normalize_domain(domain)
|
||||
payload = request.get_json(silent=True) or {}
|
||||
bucket = (payload.get("bucket") or "").strip()
|
||||
if not bucket:
|
||||
return _json_error("ValidationError", "bucket is required", 400)
|
||||
storage = _storage()
|
||||
if not storage.bucket_exists(bucket):
|
||||
return _json_error("NoSuchBucket", f"Bucket '{bucket}' does not exist", 404)
|
||||
store = _website_domains()
|
||||
if not store.get_bucket(domain):
|
||||
return _json_error("NotFound", f"No mapping found for domain '{domain}'", 404)
|
||||
store.set_mapping(domain, bucket)
|
||||
logger.info("Website domain mapping updated: %s -> %s", domain, bucket)
|
||||
return jsonify({"domain": domain, "bucket": bucket})
|
||||
|
||||
|
||||
@admin_api_bp.route("/website-domains/<domain>", methods=["DELETE"])
|
||||
@limiter.limit(lambda: _get_admin_rate_limit())
|
||||
def delete_website_domain(domain: str):
|
||||
principal, error = _require_admin()
|
||||
if error:
|
||||
return error
|
||||
if not current_app.config.get("WEBSITE_HOSTING_ENABLED", False):
|
||||
return _json_error("InvalidRequest", "Website hosting is not enabled", 400)
|
||||
domain = normalize_domain(domain)
|
||||
if not _website_domains().delete_mapping(domain):
|
||||
return _json_error("NotFound", f"No mapping found for domain '{domain}'", 404)
|
||||
logger.info("Website domain mapping deleted: %s", domain)
|
||||
return Response(status=204)
|
||||
@@ -1,24 +1,88 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import ipaddress
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
from fnmatch import fnmatch, translate
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterable, List, Optional, Pattern, Sequence, Tuple
|
||||
|
||||
|
||||
RESOURCE_PREFIX = "arn:aws:s3:::"
|
||||
|
||||
|
||||
@lru_cache(maxsize=256)
|
||||
def _compile_pattern(pattern: str) -> Pattern[str]:
|
||||
return re.compile(translate(pattern), re.IGNORECASE)
|
||||
|
||||
|
||||
def _match_string_like(value: str, pattern: str) -> bool:
|
||||
compiled = _compile_pattern(pattern)
|
||||
return bool(compiled.match(value))
|
||||
|
||||
|
||||
def _ip_in_cidr(ip_str: str, cidr: str) -> bool:
|
||||
try:
|
||||
ip = ipaddress.ip_address(ip_str)
|
||||
network = ipaddress.ip_network(cidr, strict=False)
|
||||
return ip in network
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def _evaluate_condition_operator(
|
||||
operator: str,
|
||||
condition_key: str,
|
||||
condition_values: List[str],
|
||||
context: Dict[str, Any],
|
||||
) -> bool:
|
||||
context_value = context.get(condition_key)
|
||||
op_lower = operator.lower()
|
||||
if_exists = op_lower.endswith("ifexists")
|
||||
if if_exists:
|
||||
op_lower = op_lower[:-8]
|
||||
|
||||
if context_value is None:
|
||||
return if_exists
|
||||
|
||||
context_value_str = str(context_value)
|
||||
context_value_lower = context_value_str.lower()
|
||||
|
||||
if op_lower == "stringequals":
|
||||
return context_value_str in condition_values
|
||||
elif op_lower == "stringnotequals":
|
||||
return context_value_str not in condition_values
|
||||
elif op_lower == "stringequalsignorecase":
|
||||
return context_value_lower in [v.lower() for v in condition_values]
|
||||
elif op_lower == "stringnotequalsignorecase":
|
||||
return context_value_lower not in [v.lower() for v in condition_values]
|
||||
elif op_lower == "stringlike":
|
||||
return any(_match_string_like(context_value_str, p) for p in condition_values)
|
||||
elif op_lower == "stringnotlike":
|
||||
return not any(_match_string_like(context_value_str, p) for p in condition_values)
|
||||
elif op_lower == "ipaddress":
|
||||
return any(_ip_in_cidr(context_value_str, cidr) for cidr in condition_values)
|
||||
elif op_lower == "notipaddress":
|
||||
return not any(_ip_in_cidr(context_value_str, cidr) for cidr in condition_values)
|
||||
elif op_lower == "bool":
|
||||
bool_val = context_value_lower in ("true", "1", "yes")
|
||||
return str(bool_val).lower() in [v.lower() for v in condition_values]
|
||||
elif op_lower == "null":
|
||||
is_null = context_value is None or context_value == ""
|
||||
expected_null = condition_values[0].lower() in ("true", "1", "yes") if condition_values else True
|
||||
return is_null == expected_null
|
||||
|
||||
return False
|
||||
|
||||
ACTION_ALIASES = {
|
||||
# List actions
|
||||
"s3:listbucket": "list",
|
||||
"s3:listallmybuckets": "list",
|
||||
"s3:listbucketversions": "list",
|
||||
"s3:listmultipartuploads": "list",
|
||||
"s3:listparts": "list",
|
||||
# Read actions
|
||||
"s3:getobject": "read",
|
||||
"s3:getobjectversion": "read",
|
||||
"s3:getobjecttagging": "read",
|
||||
@@ -27,7 +91,6 @@ ACTION_ALIASES = {
|
||||
"s3:getbucketversioning": "read",
|
||||
"s3:headobject": "read",
|
||||
"s3:headbucket": "read",
|
||||
# Write actions
|
||||
"s3:putobject": "write",
|
||||
"s3:createbucket": "write",
|
||||
"s3:putobjecttagging": "write",
|
||||
@@ -37,26 +100,30 @@ ACTION_ALIASES = {
|
||||
"s3:completemultipartupload": "write",
|
||||
"s3:abortmultipartupload": "write",
|
||||
"s3:copyobject": "write",
|
||||
# Delete actions
|
||||
"s3:deleteobject": "delete",
|
||||
"s3:deleteobjectversion": "delete",
|
||||
"s3:deletebucket": "delete",
|
||||
"s3:deleteobjecttagging": "delete",
|
||||
# Share actions (ACL)
|
||||
"s3:putobjectacl": "share",
|
||||
"s3:putbucketacl": "share",
|
||||
"s3:getbucketacl": "share",
|
||||
# Policy actions
|
||||
"s3:putbucketpolicy": "policy",
|
||||
"s3:getbucketpolicy": "policy",
|
||||
"s3:deletebucketpolicy": "policy",
|
||||
# Replication actions
|
||||
"s3:getreplicationconfiguration": "replication",
|
||||
"s3:putreplicationconfiguration": "replication",
|
||||
"s3:deletereplicationconfiguration": "replication",
|
||||
"s3:replicateobject": "replication",
|
||||
"s3:replicatetags": "replication",
|
||||
"s3:replicatedelete": "replication",
|
||||
"s3:getlifecycleconfiguration": "lifecycle",
|
||||
"s3:putlifecycleconfiguration": "lifecycle",
|
||||
"s3:deletelifecycleconfiguration": "lifecycle",
|
||||
"s3:getbucketlifecycle": "lifecycle",
|
||||
"s3:putbucketlifecycle": "lifecycle",
|
||||
"s3:getbucketcors": "cors",
|
||||
"s3:putbucketcors": "cors",
|
||||
"s3:deletebucketcors": "cors",
|
||||
}
|
||||
|
||||
|
||||
@@ -135,18 +202,16 @@ class BucketPolicyStatement:
|
||||
principals: List[str] | str
|
||||
actions: List[str]
|
||||
resources: List[Tuple[str | None, str | None]]
|
||||
# Performance: Pre-compiled regex patterns for resource matching
|
||||
conditions: Dict[str, Dict[str, List[str]]] = field(default_factory=dict)
|
||||
_compiled_patterns: List[Tuple[str | None, Optional[Pattern[str]]]] | None = None
|
||||
|
||||
def _get_compiled_patterns(self) -> List[Tuple[str | None, Optional[Pattern[str]]]]:
|
||||
"""Lazily compile fnmatch patterns to regex for faster matching."""
|
||||
if self._compiled_patterns is None:
|
||||
self._compiled_patterns = []
|
||||
for resource_bucket, key_pattern in self.resources:
|
||||
if key_pattern is None:
|
||||
self._compiled_patterns.append((resource_bucket, None))
|
||||
else:
|
||||
# Convert fnmatch pattern to regex
|
||||
regex_pattern = translate(key_pattern)
|
||||
self._compiled_patterns.append((resource_bucket, re.compile(regex_pattern)))
|
||||
return self._compiled_patterns
|
||||
@@ -173,11 +238,21 @@ class BucketPolicyStatement:
|
||||
if not key:
|
||||
return True
|
||||
continue
|
||||
# Performance: Use pre-compiled regex instead of fnmatch
|
||||
if compiled_pattern.match(key):
|
||||
return True
|
||||
return False
|
||||
|
||||
def matches_condition(self, context: Optional[Dict[str, Any]]) -> bool:
|
||||
if not self.conditions:
|
||||
return True
|
||||
if context is None:
|
||||
context = {}
|
||||
for operator, key_values in self.conditions.items():
|
||||
for condition_key, condition_values in key_values.items():
|
||||
if not _evaluate_condition_operator(operator, condition_key, condition_values, context):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
class BucketPolicyStore:
|
||||
"""Loads bucket policies from disk and evaluates statements."""
|
||||
@@ -219,6 +294,7 @@ class BucketPolicyStore:
|
||||
bucket: Optional[str],
|
||||
object_key: Optional[str],
|
||||
action: str,
|
||||
context: Optional[Dict[str, Any]] = None,
|
||||
) -> str | None:
|
||||
bucket = (bucket or "").lower()
|
||||
statements = self._policies.get(bucket) or []
|
||||
@@ -230,6 +306,8 @@ class BucketPolicyStore:
|
||||
continue
|
||||
if not statement.matches_resource(bucket, object_key):
|
||||
continue
|
||||
if not statement.matches_condition(context):
|
||||
continue
|
||||
if statement.effect == "deny":
|
||||
return "deny"
|
||||
decision = "allow"
|
||||
@@ -294,6 +372,7 @@ class BucketPolicyStore:
|
||||
if not resources:
|
||||
continue
|
||||
effect = statement.get("Effect", "Allow").lower()
|
||||
conditions = self._normalize_conditions(statement.get("Condition", {}))
|
||||
statements.append(
|
||||
BucketPolicyStatement(
|
||||
sid=statement.get("Sid"),
|
||||
@@ -301,6 +380,24 @@ class BucketPolicyStore:
|
||||
principals=principals,
|
||||
actions=actions or ["*"],
|
||||
resources=resources,
|
||||
conditions=conditions,
|
||||
)
|
||||
)
|
||||
return statements
|
||||
return statements
|
||||
|
||||
def _normalize_conditions(self, condition_block: Dict[str, Any]) -> Dict[str, Dict[str, List[str]]]:
|
||||
if not condition_block or not isinstance(condition_block, dict):
|
||||
return {}
|
||||
normalized: Dict[str, Dict[str, List[str]]] = {}
|
||||
for operator, key_values in condition_block.items():
|
||||
if not isinstance(key_values, dict):
|
||||
continue
|
||||
normalized[operator] = {}
|
||||
for cond_key, cond_values in key_values.items():
|
||||
if isinstance(cond_values, str):
|
||||
normalized[operator][cond_key] = [cond_values]
|
||||
elif isinstance(cond_values, list):
|
||||
normalized[operator][cond_key] = [str(v) for v in cond_values]
|
||||
else:
|
||||
normalized[operator][cond_key] = [str(cond_values)]
|
||||
return normalized
|
||||
109
app/compression.py
Normal file
109
app/compression.py
Normal file
@@ -0,0 +1,109 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import gzip
|
||||
import io
|
||||
from typing import Callable, Iterable, List, Tuple
|
||||
|
||||
COMPRESSIBLE_MIMES = frozenset([
|
||||
'application/json',
|
||||
'application/javascript',
|
||||
'application/xml',
|
||||
'text/html',
|
||||
'text/css',
|
||||
'text/plain',
|
||||
'text/xml',
|
||||
'text/javascript',
|
||||
'application/x-ndjson',
|
||||
])
|
||||
|
||||
MIN_SIZE_FOR_COMPRESSION = 500
|
||||
|
||||
|
||||
class GzipMiddleware:
|
||||
def __init__(self, app: Callable, compression_level: int = 6, min_size: int = MIN_SIZE_FOR_COMPRESSION):
|
||||
self.app = app
|
||||
self.compression_level = compression_level
|
||||
self.min_size = min_size
|
||||
|
||||
def __call__(self, environ: dict, start_response: Callable) -> Iterable[bytes]:
|
||||
accept_encoding = environ.get('HTTP_ACCEPT_ENCODING', '')
|
||||
if 'gzip' not in accept_encoding.lower():
|
||||
return self.app(environ, start_response)
|
||||
|
||||
response_started = False
|
||||
status_code = None
|
||||
response_headers: List[Tuple[str, str]] = []
|
||||
content_type = None
|
||||
content_length = None
|
||||
should_compress = False
|
||||
passthrough = False
|
||||
exc_info_holder = [None]
|
||||
|
||||
def custom_start_response(status: str, headers: List[Tuple[str, str]], exc_info=None):
|
||||
nonlocal response_started, status_code, response_headers, content_type, content_length, should_compress, passthrough
|
||||
response_started = True
|
||||
status_code = int(status.split(' ', 1)[0])
|
||||
response_headers = list(headers)
|
||||
exc_info_holder[0] = exc_info
|
||||
|
||||
for name, value in headers:
|
||||
name_lower = name.lower()
|
||||
if name_lower == 'content-type':
|
||||
content_type = value.split(';')[0].strip().lower()
|
||||
elif name_lower == 'content-length':
|
||||
try:
|
||||
content_length = int(value)
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
elif name_lower == 'content-encoding':
|
||||
passthrough = True
|
||||
return start_response(status, headers, exc_info)
|
||||
elif name_lower == 'x-stream-response':
|
||||
passthrough = True
|
||||
return start_response(status, headers, exc_info)
|
||||
|
||||
if content_type and content_type in COMPRESSIBLE_MIMES:
|
||||
if content_length is None or content_length >= self.min_size:
|
||||
should_compress = True
|
||||
else:
|
||||
passthrough = True
|
||||
return start_response(status, headers, exc_info)
|
||||
|
||||
return None
|
||||
|
||||
app_iter = self.app(environ, custom_start_response)
|
||||
|
||||
if passthrough:
|
||||
return app_iter
|
||||
|
||||
response_body = b''.join(app_iter)
|
||||
|
||||
if not response_started:
|
||||
return [response_body]
|
||||
|
||||
if should_compress and len(response_body) >= self.min_size:
|
||||
buf = io.BytesIO()
|
||||
with gzip.GzipFile(fileobj=buf, mode='wb', compresslevel=self.compression_level) as gz:
|
||||
gz.write(response_body)
|
||||
compressed = buf.getvalue()
|
||||
|
||||
if len(compressed) < len(response_body):
|
||||
response_body = compressed
|
||||
new_headers = []
|
||||
for name, value in response_headers:
|
||||
if name.lower() not in ('content-length', 'content-encoding'):
|
||||
new_headers.append((name, value))
|
||||
new_headers.append(('Content-Encoding', 'gzip'))
|
||||
new_headers.append(('Content-Length', str(len(response_body))))
|
||||
new_headers.append(('Vary', 'Accept-Encoding'))
|
||||
response_headers = new_headers
|
||||
|
||||
status_str = f"{status_code} " + {
|
||||
200: "OK", 201: "Created", 204: "No Content", 206: "Partial Content",
|
||||
301: "Moved Permanently", 302: "Found", 304: "Not Modified",
|
||||
400: "Bad Request", 401: "Unauthorized", 403: "Forbidden", 404: "Not Found",
|
||||
405: "Method Not Allowed", 409: "Conflict", 500: "Internal Server Error",
|
||||
}.get(status_code, "Unknown")
|
||||
|
||||
start_response(status_str, response_headers, exc_info_holder[0])
|
||||
return [response_body]
|
||||
294
app/config.py
294
app/config.py
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
import shutil
|
||||
import sys
|
||||
@@ -9,6 +10,30 @@ from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
import psutil
|
||||
|
||||
|
||||
def _calculate_auto_threads() -> int:
|
||||
cpu_count = psutil.cpu_count(logical=True) or 4
|
||||
return max(1, min(cpu_count * 2, 64))
|
||||
|
||||
|
||||
def _calculate_auto_connection_limit() -> int:
|
||||
available_mb = psutil.virtual_memory().available / (1024 * 1024)
|
||||
calculated = int(available_mb / 5)
|
||||
return max(20, min(calculated, 1000))
|
||||
|
||||
|
||||
def _calculate_auto_backlog(connection_limit: int) -> int:
|
||||
return max(64, min(connection_limit * 2, 4096))
|
||||
|
||||
|
||||
def _validate_rate_limit(value: str) -> str:
|
||||
pattern = r"^\d+\s+per\s+(second|minute|hour|day)$"
|
||||
if not re.match(pattern, value):
|
||||
raise ValueError(f"Invalid rate limit format: {value}. Expected format: '200 per minute'")
|
||||
return value
|
||||
|
||||
if getattr(sys, "frozen", False):
|
||||
# Running in a PyInstaller bundle
|
||||
PROJECT_ROOT = Path(sys._MEIPASS)
|
||||
@@ -55,6 +80,10 @@ class AppConfig:
|
||||
log_backup_count: int
|
||||
ratelimit_default: str
|
||||
ratelimit_storage_uri: str
|
||||
ratelimit_list_buckets: str
|
||||
ratelimit_bucket_ops: str
|
||||
ratelimit_object_ops: str
|
||||
ratelimit_head_ops: str
|
||||
cors_origins: list[str]
|
||||
cors_methods: list[str]
|
||||
cors_allow_headers: list[str]
|
||||
@@ -67,6 +96,7 @@ class AppConfig:
|
||||
stream_chunk_size: int
|
||||
multipart_min_part_size: int
|
||||
bucket_stats_cache_ttl: int
|
||||
object_cache_ttl: int
|
||||
encryption_enabled: bool
|
||||
encryption_master_key_path: Path
|
||||
kms_enabled: bool
|
||||
@@ -75,6 +105,51 @@ class AppConfig:
|
||||
display_timezone: str
|
||||
lifecycle_enabled: bool
|
||||
lifecycle_interval_seconds: int
|
||||
metrics_history_enabled: bool
|
||||
metrics_history_retention_hours: int
|
||||
metrics_history_interval_minutes: int
|
||||
operation_metrics_enabled: bool
|
||||
operation_metrics_interval_minutes: int
|
||||
operation_metrics_retention_hours: int
|
||||
server_threads: int
|
||||
server_connection_limit: int
|
||||
server_backlog: int
|
||||
server_channel_timeout: int
|
||||
server_threads_auto: bool
|
||||
server_connection_limit_auto: bool
|
||||
server_backlog_auto: bool
|
||||
site_sync_enabled: bool
|
||||
site_sync_interval_seconds: int
|
||||
site_sync_batch_size: int
|
||||
sigv4_timestamp_tolerance_seconds: int
|
||||
presigned_url_min_expiry_seconds: int
|
||||
presigned_url_max_expiry_seconds: int
|
||||
replication_connect_timeout_seconds: int
|
||||
replication_read_timeout_seconds: int
|
||||
replication_max_retries: int
|
||||
replication_streaming_threshold_bytes: int
|
||||
replication_max_failures_per_bucket: int
|
||||
site_sync_connect_timeout_seconds: int
|
||||
site_sync_read_timeout_seconds: int
|
||||
site_sync_max_retries: int
|
||||
site_sync_clock_skew_tolerance_seconds: float
|
||||
object_key_max_length_bytes: int
|
||||
object_cache_max_size: int
|
||||
bucket_config_cache_ttl_seconds: float
|
||||
object_tag_limit: int
|
||||
encryption_chunk_size_bytes: int
|
||||
kms_generate_data_key_min_bytes: int
|
||||
kms_generate_data_key_max_bytes: int
|
||||
lifecycle_max_history_per_bucket: int
|
||||
site_id: Optional[str]
|
||||
site_endpoint: Optional[str]
|
||||
site_region: str
|
||||
site_priority: int
|
||||
ratelimit_admin: str
|
||||
num_trusted_proxies: int
|
||||
allowed_redirect_hosts: list[str]
|
||||
allow_internal_endpoints: bool
|
||||
website_hosting_enabled: bool
|
||||
|
||||
@classmethod
|
||||
def from_env(cls, overrides: Optional[Dict[str, Any]] = None) -> "AppConfig":
|
||||
@@ -147,8 +222,12 @@ class AppConfig:
|
||||
log_path = log_dir / str(_get("LOG_FILE", "app.log"))
|
||||
log_max_bytes = int(_get("LOG_MAX_BYTES", 5 * 1024 * 1024))
|
||||
log_backup_count = int(_get("LOG_BACKUP_COUNT", 3))
|
||||
ratelimit_default = str(_get("RATE_LIMIT_DEFAULT", "200 per minute"))
|
||||
ratelimit_default = _validate_rate_limit(str(_get("RATE_LIMIT_DEFAULT", "200 per minute")))
|
||||
ratelimit_storage_uri = str(_get("RATE_LIMIT_STORAGE_URI", "memory://"))
|
||||
ratelimit_list_buckets = _validate_rate_limit(str(_get("RATE_LIMIT_LIST_BUCKETS", "60 per minute")))
|
||||
ratelimit_bucket_ops = _validate_rate_limit(str(_get("RATE_LIMIT_BUCKET_OPS", "120 per minute")))
|
||||
ratelimit_object_ops = _validate_rate_limit(str(_get("RATE_LIMIT_OBJECT_OPS", "240 per minute")))
|
||||
ratelimit_head_ops = _validate_rate_limit(str(_get("RATE_LIMIT_HEAD_OPS", "100 per minute")))
|
||||
|
||||
def _csv(value: str, default: list[str]) -> list[str]:
|
||||
if not value:
|
||||
@@ -161,8 +240,9 @@ class AppConfig:
|
||||
cors_allow_headers = _csv(str(_get("CORS_ALLOW_HEADERS", "*")), ["*"])
|
||||
cors_expose_headers = _csv(str(_get("CORS_EXPOSE_HEADERS", "*")), ["*"])
|
||||
session_lifetime_days = int(_get("SESSION_LIFETIME_DAYS", 30))
|
||||
bucket_stats_cache_ttl = int(_get("BUCKET_STATS_CACHE_TTL", 60))
|
||||
|
||||
bucket_stats_cache_ttl = int(_get("BUCKET_STATS_CACHE_TTL", 60))
|
||||
object_cache_ttl = int(_get("OBJECT_CACHE_TTL", 60))
|
||||
|
||||
encryption_enabled = str(_get("ENCRYPTION_ENABLED", "0")).lower() in {"1", "true", "yes", "on"}
|
||||
encryption_keys_dir = storage_root / ".myfsio.sys" / "keys"
|
||||
encryption_master_key_path = Path(_get("ENCRYPTION_MASTER_KEY_PATH", encryption_keys_dir / "master.key")).resolve()
|
||||
@@ -170,6 +250,75 @@ class AppConfig:
|
||||
kms_keys_path = Path(_get("KMS_KEYS_PATH", encryption_keys_dir / "kms_keys.json")).resolve()
|
||||
default_encryption_algorithm = str(_get("DEFAULT_ENCRYPTION_ALGORITHM", "AES256"))
|
||||
display_timezone = str(_get("DISPLAY_TIMEZONE", "UTC"))
|
||||
metrics_history_enabled = str(_get("METRICS_HISTORY_ENABLED", "0")).lower() in {"1", "true", "yes", "on"}
|
||||
metrics_history_retention_hours = int(_get("METRICS_HISTORY_RETENTION_HOURS", 24))
|
||||
metrics_history_interval_minutes = int(_get("METRICS_HISTORY_INTERVAL_MINUTES", 5))
|
||||
operation_metrics_enabled = str(_get("OPERATION_METRICS_ENABLED", "0")).lower() in {"1", "true", "yes", "on"}
|
||||
operation_metrics_interval_minutes = int(_get("OPERATION_METRICS_INTERVAL_MINUTES", 5))
|
||||
operation_metrics_retention_hours = int(_get("OPERATION_METRICS_RETENTION_HOURS", 24))
|
||||
|
||||
_raw_threads = int(_get("SERVER_THREADS", 0))
|
||||
if _raw_threads == 0:
|
||||
server_threads = _calculate_auto_threads()
|
||||
server_threads_auto = True
|
||||
else:
|
||||
server_threads = _raw_threads
|
||||
server_threads_auto = False
|
||||
|
||||
_raw_conn_limit = int(_get("SERVER_CONNECTION_LIMIT", 0))
|
||||
if _raw_conn_limit == 0:
|
||||
server_connection_limit = _calculate_auto_connection_limit()
|
||||
server_connection_limit_auto = True
|
||||
else:
|
||||
server_connection_limit = _raw_conn_limit
|
||||
server_connection_limit_auto = False
|
||||
|
||||
_raw_backlog = int(_get("SERVER_BACKLOG", 0))
|
||||
if _raw_backlog == 0:
|
||||
server_backlog = _calculate_auto_backlog(server_connection_limit)
|
||||
server_backlog_auto = True
|
||||
else:
|
||||
server_backlog = _raw_backlog
|
||||
server_backlog_auto = False
|
||||
|
||||
server_channel_timeout = int(_get("SERVER_CHANNEL_TIMEOUT", 120))
|
||||
site_sync_enabled = str(_get("SITE_SYNC_ENABLED", "0")).lower() in {"1", "true", "yes", "on"}
|
||||
site_sync_interval_seconds = int(_get("SITE_SYNC_INTERVAL_SECONDS", 60))
|
||||
site_sync_batch_size = int(_get("SITE_SYNC_BATCH_SIZE", 100))
|
||||
|
||||
sigv4_timestamp_tolerance_seconds = int(_get("SIGV4_TIMESTAMP_TOLERANCE_SECONDS", 900))
|
||||
presigned_url_min_expiry_seconds = int(_get("PRESIGNED_URL_MIN_EXPIRY_SECONDS", 1))
|
||||
presigned_url_max_expiry_seconds = int(_get("PRESIGNED_URL_MAX_EXPIRY_SECONDS", 604800))
|
||||
replication_connect_timeout_seconds = int(_get("REPLICATION_CONNECT_TIMEOUT_SECONDS", 5))
|
||||
replication_read_timeout_seconds = int(_get("REPLICATION_READ_TIMEOUT_SECONDS", 30))
|
||||
replication_max_retries = int(_get("REPLICATION_MAX_RETRIES", 2))
|
||||
replication_streaming_threshold_bytes = int(_get("REPLICATION_STREAMING_THRESHOLD_BYTES", 10 * 1024 * 1024))
|
||||
replication_max_failures_per_bucket = int(_get("REPLICATION_MAX_FAILURES_PER_BUCKET", 50))
|
||||
site_sync_connect_timeout_seconds = int(_get("SITE_SYNC_CONNECT_TIMEOUT_SECONDS", 10))
|
||||
site_sync_read_timeout_seconds = int(_get("SITE_SYNC_READ_TIMEOUT_SECONDS", 120))
|
||||
site_sync_max_retries = int(_get("SITE_SYNC_MAX_RETRIES", 2))
|
||||
site_sync_clock_skew_tolerance_seconds = float(_get("SITE_SYNC_CLOCK_SKEW_TOLERANCE_SECONDS", 1.0))
|
||||
object_key_max_length_bytes = int(_get("OBJECT_KEY_MAX_LENGTH_BYTES", 1024))
|
||||
object_cache_max_size = int(_get("OBJECT_CACHE_MAX_SIZE", 100))
|
||||
bucket_config_cache_ttl_seconds = float(_get("BUCKET_CONFIG_CACHE_TTL_SECONDS", 30.0))
|
||||
object_tag_limit = int(_get("OBJECT_TAG_LIMIT", 50))
|
||||
encryption_chunk_size_bytes = int(_get("ENCRYPTION_CHUNK_SIZE_BYTES", 64 * 1024))
|
||||
kms_generate_data_key_min_bytes = int(_get("KMS_GENERATE_DATA_KEY_MIN_BYTES", 1))
|
||||
kms_generate_data_key_max_bytes = int(_get("KMS_GENERATE_DATA_KEY_MAX_BYTES", 1024))
|
||||
lifecycle_max_history_per_bucket = int(_get("LIFECYCLE_MAX_HISTORY_PER_BUCKET", 50))
|
||||
|
||||
site_id_raw = _get("SITE_ID", None)
|
||||
site_id = str(site_id_raw).strip() if site_id_raw else None
|
||||
site_endpoint_raw = _get("SITE_ENDPOINT", None)
|
||||
site_endpoint = str(site_endpoint_raw).strip() if site_endpoint_raw else None
|
||||
site_region = str(_get("SITE_REGION", "us-east-1"))
|
||||
site_priority = int(_get("SITE_PRIORITY", 100))
|
||||
ratelimit_admin = _validate_rate_limit(str(_get("RATE_LIMIT_ADMIN", "60 per minute")))
|
||||
num_trusted_proxies = int(_get("NUM_TRUSTED_PROXIES", 1))
|
||||
allowed_redirect_hosts_raw = _get("ALLOWED_REDIRECT_HOSTS", "")
|
||||
allowed_redirect_hosts = [h.strip() for h in str(allowed_redirect_hosts_raw).split(",") if h.strip()]
|
||||
allow_internal_endpoints = str(_get("ALLOW_INTERNAL_ENDPOINTS", "0")).lower() in {"1", "true", "yes", "on"}
|
||||
website_hosting_enabled = str(_get("WEBSITE_HOSTING_ENABLED", "0")).lower() in {"1", "true", "yes", "on"}
|
||||
|
||||
return cls(storage_root=storage_root,
|
||||
max_upload_size=max_upload_size,
|
||||
@@ -188,6 +337,10 @@ class AppConfig:
|
||||
log_backup_count=log_backup_count,
|
||||
ratelimit_default=ratelimit_default,
|
||||
ratelimit_storage_uri=ratelimit_storage_uri,
|
||||
ratelimit_list_buckets=ratelimit_list_buckets,
|
||||
ratelimit_bucket_ops=ratelimit_bucket_ops,
|
||||
ratelimit_object_ops=ratelimit_object_ops,
|
||||
ratelimit_head_ops=ratelimit_head_ops,
|
||||
cors_origins=cors_origins,
|
||||
cors_methods=cors_methods,
|
||||
cors_allow_headers=cors_allow_headers,
|
||||
@@ -200,6 +353,7 @@ class AppConfig:
|
||||
stream_chunk_size=stream_chunk_size,
|
||||
multipart_min_part_size=multipart_min_part_size,
|
||||
bucket_stats_cache_ttl=bucket_stats_cache_ttl,
|
||||
object_cache_ttl=object_cache_ttl,
|
||||
encryption_enabled=encryption_enabled,
|
||||
encryption_master_key_path=encryption_master_key_path,
|
||||
kms_enabled=kms_enabled,
|
||||
@@ -207,7 +361,52 @@ class AppConfig:
|
||||
default_encryption_algorithm=default_encryption_algorithm,
|
||||
display_timezone=display_timezone,
|
||||
lifecycle_enabled=lifecycle_enabled,
|
||||
lifecycle_interval_seconds=lifecycle_interval_seconds)
|
||||
lifecycle_interval_seconds=lifecycle_interval_seconds,
|
||||
metrics_history_enabled=metrics_history_enabled,
|
||||
metrics_history_retention_hours=metrics_history_retention_hours,
|
||||
metrics_history_interval_minutes=metrics_history_interval_minutes,
|
||||
operation_metrics_enabled=operation_metrics_enabled,
|
||||
operation_metrics_interval_minutes=operation_metrics_interval_minutes,
|
||||
operation_metrics_retention_hours=operation_metrics_retention_hours,
|
||||
server_threads=server_threads,
|
||||
server_connection_limit=server_connection_limit,
|
||||
server_backlog=server_backlog,
|
||||
server_channel_timeout=server_channel_timeout,
|
||||
server_threads_auto=server_threads_auto,
|
||||
server_connection_limit_auto=server_connection_limit_auto,
|
||||
server_backlog_auto=server_backlog_auto,
|
||||
site_sync_enabled=site_sync_enabled,
|
||||
site_sync_interval_seconds=site_sync_interval_seconds,
|
||||
site_sync_batch_size=site_sync_batch_size,
|
||||
sigv4_timestamp_tolerance_seconds=sigv4_timestamp_tolerance_seconds,
|
||||
presigned_url_min_expiry_seconds=presigned_url_min_expiry_seconds,
|
||||
presigned_url_max_expiry_seconds=presigned_url_max_expiry_seconds,
|
||||
replication_connect_timeout_seconds=replication_connect_timeout_seconds,
|
||||
replication_read_timeout_seconds=replication_read_timeout_seconds,
|
||||
replication_max_retries=replication_max_retries,
|
||||
replication_streaming_threshold_bytes=replication_streaming_threshold_bytes,
|
||||
replication_max_failures_per_bucket=replication_max_failures_per_bucket,
|
||||
site_sync_connect_timeout_seconds=site_sync_connect_timeout_seconds,
|
||||
site_sync_read_timeout_seconds=site_sync_read_timeout_seconds,
|
||||
site_sync_max_retries=site_sync_max_retries,
|
||||
site_sync_clock_skew_tolerance_seconds=site_sync_clock_skew_tolerance_seconds,
|
||||
object_key_max_length_bytes=object_key_max_length_bytes,
|
||||
object_cache_max_size=object_cache_max_size,
|
||||
bucket_config_cache_ttl_seconds=bucket_config_cache_ttl_seconds,
|
||||
object_tag_limit=object_tag_limit,
|
||||
encryption_chunk_size_bytes=encryption_chunk_size_bytes,
|
||||
kms_generate_data_key_min_bytes=kms_generate_data_key_min_bytes,
|
||||
kms_generate_data_key_max_bytes=kms_generate_data_key_max_bytes,
|
||||
lifecycle_max_history_per_bucket=lifecycle_max_history_per_bucket,
|
||||
site_id=site_id,
|
||||
site_endpoint=site_endpoint,
|
||||
site_region=site_region,
|
||||
site_priority=site_priority,
|
||||
ratelimit_admin=ratelimit_admin,
|
||||
num_trusted_proxies=num_trusted_proxies,
|
||||
allowed_redirect_hosts=allowed_redirect_hosts,
|
||||
allow_internal_endpoints=allow_internal_endpoints,
|
||||
website_hosting_enabled=website_hosting_enabled)
|
||||
|
||||
def validate_and_report(self) -> list[str]:
|
||||
"""Validate configuration and return a list of warnings/issues.
|
||||
@@ -267,7 +466,35 @@ class AppConfig:
|
||||
|
||||
if "*" in self.cors_origins:
|
||||
issues.append("INFO: CORS_ORIGINS is set to '*'. Consider restricting to specific domains in production.")
|
||||
|
||||
|
||||
if not (1 <= self.server_threads <= 64):
|
||||
issues.append(f"CRITICAL: SERVER_THREADS={self.server_threads} is outside valid range (1-64). Server cannot start.")
|
||||
if not (10 <= self.server_connection_limit <= 1000):
|
||||
issues.append(f"CRITICAL: SERVER_CONNECTION_LIMIT={self.server_connection_limit} is outside valid range (10-1000). Server cannot start.")
|
||||
if not (64 <= self.server_backlog <= 4096):
|
||||
issues.append(f"CRITICAL: SERVER_BACKLOG={self.server_backlog} is outside valid range (64-4096). Server cannot start.")
|
||||
if not (10 <= self.server_channel_timeout <= 300):
|
||||
issues.append(f"CRITICAL: SERVER_CHANNEL_TIMEOUT={self.server_channel_timeout} is outside valid range (10-300). Server cannot start.")
|
||||
|
||||
if sys.platform != "win32":
|
||||
try:
|
||||
import resource
|
||||
soft_limit, _ = resource.getrlimit(resource.RLIMIT_NOFILE)
|
||||
threshold = int(soft_limit * 0.8)
|
||||
if self.server_connection_limit > threshold:
|
||||
issues.append(f"WARNING: SERVER_CONNECTION_LIMIT={self.server_connection_limit} exceeds 80% of system file descriptor limit (soft={soft_limit}). Consider running 'ulimit -n {self.server_connection_limit + 100}'.")
|
||||
except (ImportError, OSError):
|
||||
pass
|
||||
|
||||
try:
|
||||
import psutil
|
||||
available_mb = psutil.virtual_memory().available / (1024 * 1024)
|
||||
estimated_mb = self.server_threads * 50
|
||||
if estimated_mb > available_mb * 0.5:
|
||||
issues.append(f"WARNING: SERVER_THREADS={self.server_threads} may require ~{estimated_mb}MB memory, exceeding 50% of available RAM ({int(available_mb)}MB).")
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
return issues
|
||||
|
||||
def print_startup_summary(self) -> None:
|
||||
@@ -285,6 +512,14 @@ class AppConfig:
|
||||
print(f" ENCRYPTION: Enabled (Master key: {self.encryption_master_key_path})")
|
||||
if self.kms_enabled:
|
||||
print(f" KMS: Enabled (Keys: {self.kms_keys_path})")
|
||||
if self.website_hosting_enabled:
|
||||
print(f" WEBSITE_HOSTING: Enabled")
|
||||
def _auto(flag: bool) -> str:
|
||||
return " (auto)" if flag else ""
|
||||
print(f" SERVER_THREADS: {self.server_threads}{_auto(self.server_threads_auto)}")
|
||||
print(f" CONNECTION_LIMIT: {self.server_connection_limit}{_auto(self.server_connection_limit_auto)}")
|
||||
print(f" BACKLOG: {self.server_backlog}{_auto(self.server_backlog_auto)}")
|
||||
print(f" CHANNEL_TIMEOUT: {self.server_channel_timeout}s")
|
||||
print("=" * 60)
|
||||
|
||||
issues = self.validate_and_report()
|
||||
@@ -315,6 +550,7 @@ class AppConfig:
|
||||
"STREAM_CHUNK_SIZE": self.stream_chunk_size,
|
||||
"MULTIPART_MIN_PART_SIZE": self.multipart_min_part_size,
|
||||
"BUCKET_STATS_CACHE_TTL": self.bucket_stats_cache_ttl,
|
||||
"OBJECT_CACHE_TTL": self.object_cache_ttl,
|
||||
"LOG_LEVEL": self.log_level,
|
||||
"LOG_TO_FILE": self.log_to_file,
|
||||
"LOG_FILE": str(self.log_path),
|
||||
@@ -322,6 +558,10 @@ class AppConfig:
|
||||
"LOG_BACKUP_COUNT": self.log_backup_count,
|
||||
"RATELIMIT_DEFAULT": self.ratelimit_default,
|
||||
"RATELIMIT_STORAGE_URI": self.ratelimit_storage_uri,
|
||||
"RATELIMIT_LIST_BUCKETS": self.ratelimit_list_buckets,
|
||||
"RATELIMIT_BUCKET_OPS": self.ratelimit_bucket_ops,
|
||||
"RATELIMIT_OBJECT_OPS": self.ratelimit_object_ops,
|
||||
"RATELIMIT_HEAD_OPS": self.ratelimit_head_ops,
|
||||
"CORS_ORIGINS": self.cors_origins,
|
||||
"CORS_METHODS": self.cors_methods,
|
||||
"CORS_ALLOW_HEADERS": self.cors_allow_headers,
|
||||
@@ -333,4 +573,48 @@ class AppConfig:
|
||||
"KMS_KEYS_PATH": str(self.kms_keys_path),
|
||||
"DEFAULT_ENCRYPTION_ALGORITHM": self.default_encryption_algorithm,
|
||||
"DISPLAY_TIMEZONE": self.display_timezone,
|
||||
"LIFECYCLE_ENABLED": self.lifecycle_enabled,
|
||||
"LIFECYCLE_INTERVAL_SECONDS": self.lifecycle_interval_seconds,
|
||||
"METRICS_HISTORY_ENABLED": self.metrics_history_enabled,
|
||||
"METRICS_HISTORY_RETENTION_HOURS": self.metrics_history_retention_hours,
|
||||
"METRICS_HISTORY_INTERVAL_MINUTES": self.metrics_history_interval_minutes,
|
||||
"OPERATION_METRICS_ENABLED": self.operation_metrics_enabled,
|
||||
"OPERATION_METRICS_INTERVAL_MINUTES": self.operation_metrics_interval_minutes,
|
||||
"OPERATION_METRICS_RETENTION_HOURS": self.operation_metrics_retention_hours,
|
||||
"SERVER_THREADS": self.server_threads,
|
||||
"SERVER_CONNECTION_LIMIT": self.server_connection_limit,
|
||||
"SERVER_BACKLOG": self.server_backlog,
|
||||
"SERVER_CHANNEL_TIMEOUT": self.server_channel_timeout,
|
||||
"SITE_SYNC_ENABLED": self.site_sync_enabled,
|
||||
"SITE_SYNC_INTERVAL_SECONDS": self.site_sync_interval_seconds,
|
||||
"SITE_SYNC_BATCH_SIZE": self.site_sync_batch_size,
|
||||
"SIGV4_TIMESTAMP_TOLERANCE_SECONDS": self.sigv4_timestamp_tolerance_seconds,
|
||||
"PRESIGNED_URL_MIN_EXPIRY_SECONDS": self.presigned_url_min_expiry_seconds,
|
||||
"PRESIGNED_URL_MAX_EXPIRY_SECONDS": self.presigned_url_max_expiry_seconds,
|
||||
"REPLICATION_CONNECT_TIMEOUT_SECONDS": self.replication_connect_timeout_seconds,
|
||||
"REPLICATION_READ_TIMEOUT_SECONDS": self.replication_read_timeout_seconds,
|
||||
"REPLICATION_MAX_RETRIES": self.replication_max_retries,
|
||||
"REPLICATION_STREAMING_THRESHOLD_BYTES": self.replication_streaming_threshold_bytes,
|
||||
"REPLICATION_MAX_FAILURES_PER_BUCKET": self.replication_max_failures_per_bucket,
|
||||
"SITE_SYNC_CONNECT_TIMEOUT_SECONDS": self.site_sync_connect_timeout_seconds,
|
||||
"SITE_SYNC_READ_TIMEOUT_SECONDS": self.site_sync_read_timeout_seconds,
|
||||
"SITE_SYNC_MAX_RETRIES": self.site_sync_max_retries,
|
||||
"SITE_SYNC_CLOCK_SKEW_TOLERANCE_SECONDS": self.site_sync_clock_skew_tolerance_seconds,
|
||||
"OBJECT_KEY_MAX_LENGTH_BYTES": self.object_key_max_length_bytes,
|
||||
"OBJECT_CACHE_MAX_SIZE": self.object_cache_max_size,
|
||||
"BUCKET_CONFIG_CACHE_TTL_SECONDS": self.bucket_config_cache_ttl_seconds,
|
||||
"OBJECT_TAG_LIMIT": self.object_tag_limit,
|
||||
"ENCRYPTION_CHUNK_SIZE_BYTES": self.encryption_chunk_size_bytes,
|
||||
"KMS_GENERATE_DATA_KEY_MIN_BYTES": self.kms_generate_data_key_min_bytes,
|
||||
"KMS_GENERATE_DATA_KEY_MAX_BYTES": self.kms_generate_data_key_max_bytes,
|
||||
"LIFECYCLE_MAX_HISTORY_PER_BUCKET": self.lifecycle_max_history_per_bucket,
|
||||
"SITE_ID": self.site_id,
|
||||
"SITE_ENDPOINT": self.site_endpoint,
|
||||
"SITE_REGION": self.site_region,
|
||||
"SITE_PRIORITY": self.site_priority,
|
||||
"RATE_LIMIT_ADMIN": self.ratelimit_admin,
|
||||
"NUM_TRUSTED_PROXIES": self.num_trusted_proxies,
|
||||
"ALLOWED_REDIRECT_HOSTS": self.allowed_redirect_hosts,
|
||||
"ALLOW_INTERNAL_ENDPOINTS": self.allow_internal_endpoints,
|
||||
"WEBSITE_HOSTING_ENABLED": self.website_hosting_enabled,
|
||||
}
|
||||
|
||||
@@ -189,7 +189,13 @@ class EncryptedObjectStorage:
|
||||
|
||||
def list_objects(self, bucket_name: str, **kwargs):
|
||||
return self.storage.list_objects(bucket_name, **kwargs)
|
||||
|
||||
|
||||
def list_objects_shallow(self, bucket_name: str, **kwargs):
|
||||
return self.storage.list_objects_shallow(bucket_name, **kwargs)
|
||||
|
||||
def search_objects(self, bucket_name: str, query: str, **kwargs):
|
||||
return self.storage.search_objects(bucket_name, query, **kwargs)
|
||||
|
||||
def list_objects_all(self, bucket_name: str):
|
||||
return self.storage.list_objects_all(bucket_name)
|
||||
|
||||
@@ -270,9 +276,15 @@ class EncryptedObjectStorage:
|
||||
|
||||
def get_bucket_quota(self, bucket_name: str):
|
||||
return self.storage.get_bucket_quota(bucket_name)
|
||||
|
||||
|
||||
def set_bucket_quota(self, bucket_name: str, *, max_bytes=None, max_objects=None):
|
||||
return self.storage.set_bucket_quota(bucket_name, max_bytes=max_bytes, max_objects=max_objects)
|
||||
|
||||
def get_bucket_website(self, bucket_name: str):
|
||||
return self.storage.get_bucket_website(bucket_name)
|
||||
|
||||
def set_bucket_website(self, bucket_name: str, website_config):
|
||||
return self.storage.set_bucket_website(bucket_name, website_config)
|
||||
|
||||
def _compute_etag(self, path: Path) -> str:
|
||||
return self.storage._compute_etag(path)
|
||||
|
||||
@@ -1,15 +1,44 @@
|
||||
"""Encryption providers for server-side and client-side encryption."""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import secrets
|
||||
import subprocess
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, BinaryIO, Dict, Generator, Optional
|
||||
|
||||
from cryptography.hazmat.primitives.ciphers.aead import AESGCM
|
||||
from cryptography.hazmat.primitives.kdf.hkdf import HKDF
|
||||
from cryptography.hazmat.primitives import hashes
|
||||
|
||||
if sys.platform != "win32":
|
||||
import fcntl
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _set_secure_file_permissions(file_path: Path) -> None:
|
||||
"""Set restrictive file permissions (owner read/write only)."""
|
||||
if sys.platform == "win32":
|
||||
try:
|
||||
username = os.environ.get("USERNAME", "")
|
||||
if username:
|
||||
subprocess.run(
|
||||
["icacls", str(file_path), "/inheritance:r",
|
||||
"/grant:r", f"{username}:F"],
|
||||
check=True, capture_output=True
|
||||
)
|
||||
else:
|
||||
logger.warning("Could not set secure permissions on %s: USERNAME not set", file_path)
|
||||
except (subprocess.SubprocessError, OSError) as exc:
|
||||
logger.warning("Failed to set secure permissions on %s: %s", file_path, exc)
|
||||
else:
|
||||
os.chmod(file_path, 0o600)
|
||||
|
||||
|
||||
class EncryptionError(Exception):
|
||||
@@ -59,22 +88,34 @@ class EncryptionMetadata:
|
||||
|
||||
class EncryptionProvider:
|
||||
"""Base class for encryption providers."""
|
||||
|
||||
|
||||
def encrypt(self, plaintext: bytes, context: Dict[str, str] | None = None) -> EncryptionResult:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def decrypt(self, ciphertext: bytes, nonce: bytes, encrypted_data_key: bytes,
|
||||
key_id: str, context: Dict[str, str] | None = None) -> bytes:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def generate_data_key(self) -> tuple[bytes, bytes]:
|
||||
"""Generate a data key and its encrypted form.
|
||||
|
||||
|
||||
Returns:
|
||||
Tuple of (plaintext_key, encrypted_key)
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def decrypt_data_key(self, encrypted_data_key: bytes, key_id: str | None = None) -> bytes:
|
||||
"""Decrypt an encrypted data key.
|
||||
|
||||
Args:
|
||||
encrypted_data_key: The encrypted data key bytes
|
||||
key_id: Optional key identifier (used by KMS providers)
|
||||
|
||||
Returns:
|
||||
The decrypted data key
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class LocalKeyEncryption(EncryptionProvider):
|
||||
"""SSE-S3 style encryption using a local master key.
|
||||
@@ -99,28 +140,48 @@ class LocalKeyEncryption(EncryptionProvider):
|
||||
return self._master_key
|
||||
|
||||
def _load_or_create_master_key(self) -> bytes:
|
||||
"""Load master key from file or generate a new one."""
|
||||
if self.master_key_path.exists():
|
||||
try:
|
||||
return base64.b64decode(self.master_key_path.read_text().strip())
|
||||
except Exception as exc:
|
||||
raise EncryptionError(f"Failed to load master key: {exc}") from exc
|
||||
|
||||
key = secrets.token_bytes(32)
|
||||
"""Load master key from file or generate a new one (with file locking)."""
|
||||
lock_path = self.master_key_path.with_suffix(".lock")
|
||||
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
self.master_key_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self.master_key_path.write_text(base64.b64encode(key).decode())
|
||||
with open(lock_path, "w") as lock_file:
|
||||
if sys.platform == "win32":
|
||||
import msvcrt
|
||||
msvcrt.locking(lock_file.fileno(), msvcrt.LK_LOCK, 1)
|
||||
else:
|
||||
fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX)
|
||||
try:
|
||||
if self.master_key_path.exists():
|
||||
try:
|
||||
return base64.b64decode(self.master_key_path.read_text().strip())
|
||||
except Exception as exc:
|
||||
raise EncryptionError(f"Failed to load master key: {exc}") from exc
|
||||
key = secrets.token_bytes(32)
|
||||
try:
|
||||
self.master_key_path.write_text(base64.b64encode(key).decode())
|
||||
_set_secure_file_permissions(self.master_key_path)
|
||||
except OSError as exc:
|
||||
raise EncryptionError(f"Failed to save master key: {exc}") from exc
|
||||
return key
|
||||
finally:
|
||||
if sys.platform == "win32":
|
||||
import msvcrt
|
||||
msvcrt.locking(lock_file.fileno(), msvcrt.LK_UNLCK, 1)
|
||||
else:
|
||||
fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
|
||||
except OSError as exc:
|
||||
raise EncryptionError(f"Failed to save master key: {exc}") from exc
|
||||
return key
|
||||
raise EncryptionError(f"Failed to acquire lock for master key: {exc}") from exc
|
||||
|
||||
DATA_KEY_AAD = b'{"purpose":"data_key","version":1}'
|
||||
|
||||
def _encrypt_data_key(self, data_key: bytes) -> bytes:
|
||||
"""Encrypt the data key with the master key."""
|
||||
aesgcm = AESGCM(self.master_key)
|
||||
nonce = secrets.token_bytes(12)
|
||||
encrypted = aesgcm.encrypt(nonce, data_key, None)
|
||||
encrypted = aesgcm.encrypt(nonce, data_key, self.DATA_KEY_AAD)
|
||||
return nonce + encrypted
|
||||
|
||||
|
||||
def _decrypt_data_key(self, encrypted_data_key: bytes) -> bytes:
|
||||
"""Decrypt the data key using the master key."""
|
||||
if len(encrypted_data_key) < 12 + 32 + 16: # nonce + key + tag
|
||||
@@ -129,10 +190,17 @@ class LocalKeyEncryption(EncryptionProvider):
|
||||
nonce = encrypted_data_key[:12]
|
||||
ciphertext = encrypted_data_key[12:]
|
||||
try:
|
||||
return aesgcm.decrypt(nonce, ciphertext, None)
|
||||
except Exception as exc:
|
||||
raise EncryptionError(f"Failed to decrypt data key: {exc}") from exc
|
||||
|
||||
return aesgcm.decrypt(nonce, ciphertext, self.DATA_KEY_AAD)
|
||||
except Exception:
|
||||
try:
|
||||
return aesgcm.decrypt(nonce, ciphertext, None)
|
||||
except Exception as exc:
|
||||
raise EncryptionError(f"Failed to decrypt data key: {exc}") from exc
|
||||
|
||||
def decrypt_data_key(self, encrypted_data_key: bytes, key_id: str | None = None) -> bytes:
|
||||
"""Decrypt an encrypted data key (key_id ignored for local encryption)."""
|
||||
return self._decrypt_data_key(encrypted_data_key)
|
||||
|
||||
def generate_data_key(self) -> tuple[bytes, bytes]:
|
||||
"""Generate a data key and its encrypted form."""
|
||||
plaintext_key = secrets.token_bytes(32)
|
||||
@@ -142,11 +210,12 @@ class LocalKeyEncryption(EncryptionProvider):
|
||||
def encrypt(self, plaintext: bytes, context: Dict[str, str] | None = None) -> EncryptionResult:
|
||||
"""Encrypt data using envelope encryption."""
|
||||
data_key, encrypted_data_key = self.generate_data_key()
|
||||
|
||||
|
||||
aesgcm = AESGCM(data_key)
|
||||
nonce = secrets.token_bytes(12)
|
||||
ciphertext = aesgcm.encrypt(nonce, plaintext, None)
|
||||
|
||||
aad = json.dumps(context, sort_keys=True).encode() if context else None
|
||||
ciphertext = aesgcm.encrypt(nonce, plaintext, aad)
|
||||
|
||||
return EncryptionResult(
|
||||
ciphertext=ciphertext,
|
||||
nonce=nonce,
|
||||
@@ -159,10 +228,11 @@ class LocalKeyEncryption(EncryptionProvider):
|
||||
"""Decrypt data using envelope encryption."""
|
||||
data_key = self._decrypt_data_key(encrypted_data_key)
|
||||
aesgcm = AESGCM(data_key)
|
||||
aad = json.dumps(context, sort_keys=True).encode() if context else None
|
||||
try:
|
||||
return aesgcm.decrypt(nonce, ciphertext, None)
|
||||
return aesgcm.decrypt(nonce, ciphertext, aad)
|
||||
except Exception as exc:
|
||||
raise EncryptionError(f"Failed to decrypt data: {exc}") from exc
|
||||
raise EncryptionError("Failed to decrypt data") from exc
|
||||
|
||||
|
||||
class StreamingEncryptor:
|
||||
@@ -180,12 +250,14 @@ class StreamingEncryptor:
|
||||
self.chunk_size = chunk_size
|
||||
|
||||
def _derive_chunk_nonce(self, base_nonce: bytes, chunk_index: int) -> bytes:
|
||||
"""Derive a unique nonce for each chunk.
|
||||
|
||||
Performance: Use direct byte manipulation instead of full int conversion.
|
||||
"""
|
||||
# Performance: Only modify last 4 bytes instead of full 12-byte conversion
|
||||
return base_nonce[:8] + (chunk_index ^ int.from_bytes(base_nonce[8:], "big")).to_bytes(4, "big")
|
||||
"""Derive a unique nonce for each chunk using HKDF."""
|
||||
hkdf = HKDF(
|
||||
algorithm=hashes.SHA256(),
|
||||
length=12,
|
||||
salt=base_nonce,
|
||||
info=chunk_index.to_bytes(4, "big"),
|
||||
)
|
||||
return hkdf.derive(b"chunk_nonce")
|
||||
|
||||
def encrypt_stream(self, stream: BinaryIO,
|
||||
context: Dict[str, str] | None = None) -> tuple[BinaryIO, EncryptionMetadata]:
|
||||
@@ -234,10 +306,7 @@ class StreamingEncryptor:
|
||||
|
||||
Performance: Writes chunks directly to output buffer instead of accumulating in list.
|
||||
"""
|
||||
if isinstance(self.provider, LocalKeyEncryption):
|
||||
data_key = self.provider._decrypt_data_key(metadata.encrypted_data_key)
|
||||
else:
|
||||
raise EncryptionError("Unsupported provider for streaming decryption")
|
||||
data_key = self.provider.decrypt_data_key(metadata.encrypted_data_key, metadata.key_id)
|
||||
|
||||
aesgcm = AESGCM(data_key)
|
||||
base_nonce = metadata.nonce
|
||||
@@ -310,7 +379,8 @@ class EncryptionManager:
|
||||
|
||||
def get_streaming_encryptor(self) -> StreamingEncryptor:
|
||||
if self._streaming_encryptor is None:
|
||||
self._streaming_encryptor = StreamingEncryptor(self.get_local_provider())
|
||||
chunk_size = self.config.get("encryption_chunk_size_bytes", 64 * 1024)
|
||||
self._streaming_encryptor = StreamingEncryptor(self.get_local_provider(), chunk_size=chunk_size)
|
||||
return self._streaming_encryptor
|
||||
|
||||
def encrypt_object(self, data: bytes, algorithm: str = "AES256",
|
||||
@@ -403,7 +473,8 @@ class SSECEncryption(EncryptionProvider):
|
||||
def encrypt(self, plaintext: bytes, context: Dict[str, str] | None = None) -> EncryptionResult:
|
||||
aesgcm = AESGCM(self.customer_key)
|
||||
nonce = secrets.token_bytes(12)
|
||||
ciphertext = aesgcm.encrypt(nonce, plaintext, None)
|
||||
aad = json.dumps(context, sort_keys=True).encode() if context else None
|
||||
ciphertext = aesgcm.encrypt(nonce, plaintext, aad)
|
||||
|
||||
return EncryptionResult(
|
||||
ciphertext=ciphertext,
|
||||
@@ -415,10 +486,11 @@ class SSECEncryption(EncryptionProvider):
|
||||
def decrypt(self, ciphertext: bytes, nonce: bytes, encrypted_data_key: bytes,
|
||||
key_id: str, context: Dict[str, str] | None = None) -> bytes:
|
||||
aesgcm = AESGCM(self.customer_key)
|
||||
aad = json.dumps(context, sort_keys=True).encode() if context else None
|
||||
try:
|
||||
return aesgcm.decrypt(nonce, ciphertext, None)
|
||||
return aesgcm.decrypt(nonce, ciphertext, aad)
|
||||
except Exception as exc:
|
||||
raise EncryptionError(f"SSE-C decryption failed: {exc}") from exc
|
||||
raise EncryptionError("SSE-C decryption failed") from exc
|
||||
|
||||
def generate_data_key(self) -> tuple[bytes, bytes]:
|
||||
return self.customer_key, b""
|
||||
@@ -472,34 +544,36 @@ class ClientEncryptionHelper:
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def encrypt_with_key(plaintext: bytes, key_b64: str) -> Dict[str, str]:
|
||||
def encrypt_with_key(plaintext: bytes, key_b64: str, context: Dict[str, str] | None = None) -> Dict[str, str]:
|
||||
"""Encrypt data with a client-provided key."""
|
||||
key = base64.b64decode(key_b64)
|
||||
if len(key) != 32:
|
||||
raise EncryptionError("Key must be 256 bits (32 bytes)")
|
||||
|
||||
|
||||
aesgcm = AESGCM(key)
|
||||
nonce = secrets.token_bytes(12)
|
||||
ciphertext = aesgcm.encrypt(nonce, plaintext, None)
|
||||
|
||||
aad = json.dumps(context, sort_keys=True).encode() if context else None
|
||||
ciphertext = aesgcm.encrypt(nonce, plaintext, aad)
|
||||
|
||||
return {
|
||||
"ciphertext": base64.b64encode(ciphertext).decode(),
|
||||
"nonce": base64.b64encode(nonce).decode(),
|
||||
"algorithm": "AES-256-GCM",
|
||||
}
|
||||
|
||||
|
||||
@staticmethod
|
||||
def decrypt_with_key(ciphertext_b64: str, nonce_b64: str, key_b64: str) -> bytes:
|
||||
def decrypt_with_key(ciphertext_b64: str, nonce_b64: str, key_b64: str, context: Dict[str, str] | None = None) -> bytes:
|
||||
"""Decrypt data with a client-provided key."""
|
||||
key = base64.b64decode(key_b64)
|
||||
nonce = base64.b64decode(nonce_b64)
|
||||
ciphertext = base64.b64decode(ciphertext_b64)
|
||||
|
||||
|
||||
if len(key) != 32:
|
||||
raise EncryptionError("Key must be 256 bits (32 bytes)")
|
||||
|
||||
|
||||
aesgcm = AESGCM(key)
|
||||
aad = json.dumps(context, sort_keys=True).encode() if context else None
|
||||
try:
|
||||
return aesgcm.decrypt(nonce, ciphertext, None)
|
||||
return aesgcm.decrypt(nonce, ciphertext, aad)
|
||||
except Exception as exc:
|
||||
raise EncryptionError(f"Decryption failed: {exc}") from exc
|
||||
raise EncryptionError("Decryption failed") from exc
|
||||
|
||||
@@ -6,6 +6,7 @@ from typing import Optional, Dict, Any
|
||||
from xml.etree.ElementTree import Element, SubElement, tostring
|
||||
|
||||
from flask import Response, jsonify, request, flash, redirect, url_for, g
|
||||
from flask_limiter import RateLimitExceeded
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -172,10 +173,22 @@ def handle_app_error(error: AppError) -> Response:
|
||||
return error.to_xml_response()
|
||||
|
||||
|
||||
def handle_rate_limit_exceeded(e: RateLimitExceeded) -> Response:
|
||||
g.s3_error_code = "SlowDown"
|
||||
error = Element("Error")
|
||||
SubElement(error, "Code").text = "SlowDown"
|
||||
SubElement(error, "Message").text = "Please reduce your request rate."
|
||||
SubElement(error, "Resource").text = request.path
|
||||
SubElement(error, "RequestId").text = getattr(g, "request_id", "")
|
||||
xml_bytes = tostring(error, encoding="utf-8")
|
||||
return Response(xml_bytes, status=429, mimetype="application/xml")
|
||||
|
||||
|
||||
def register_error_handlers(app):
|
||||
"""Register error handlers with a Flask app."""
|
||||
app.register_error_handler(AppError, handle_app_error)
|
||||
|
||||
app.register_error_handler(RateLimitExceeded, handle_rate_limit_exceeded)
|
||||
|
||||
for error_class in [
|
||||
BucketNotFoundError, BucketAlreadyExistsError, BucketNotEmptyError,
|
||||
ObjectNotFoundError, InvalidObjectKeyError,
|
||||
|
||||
165
app/iam.py
165
app/iam.py
@@ -1,8 +1,12 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import secrets
|
||||
import threading
|
||||
import time
|
||||
from collections import deque
|
||||
from dataclasses import dataclass
|
||||
@@ -15,7 +19,7 @@ class IamError(RuntimeError):
|
||||
"""Raised when authentication or authorization fails."""
|
||||
|
||||
|
||||
S3_ACTIONS = {"list", "read", "write", "delete", "share", "policy", "replication"}
|
||||
S3_ACTIONS = {"list", "read", "write", "delete", "share", "policy", "replication", "lifecycle", "cors"}
|
||||
IAM_ACTIONS = {
|
||||
"iam:list_users",
|
||||
"iam:create_user",
|
||||
@@ -71,6 +75,16 @@ ACTION_ALIASES = {
|
||||
"s3:replicateobject": "replication",
|
||||
"s3:replicatetags": "replication",
|
||||
"s3:replicatedelete": "replication",
|
||||
"lifecycle": "lifecycle",
|
||||
"s3:getlifecycleconfiguration": "lifecycle",
|
||||
"s3:putlifecycleconfiguration": "lifecycle",
|
||||
"s3:deletelifecycleconfiguration": "lifecycle",
|
||||
"s3:getbucketlifecycle": "lifecycle",
|
||||
"s3:putbucketlifecycle": "lifecycle",
|
||||
"cors": "cors",
|
||||
"s3:getbucketcors": "cors",
|
||||
"s3:putbucketcors": "cors",
|
||||
"s3:deletebucketcors": "cors",
|
||||
"iam:listusers": "iam:list_users",
|
||||
"iam:createuser": "iam:create_user",
|
||||
"iam:deleteuser": "iam:delete_user",
|
||||
@@ -107,12 +121,15 @@ class IamService:
|
||||
self._raw_config: Dict[str, Any] = {}
|
||||
self._failed_attempts: Dict[str, Deque[datetime]] = {}
|
||||
self._last_load_time = 0.0
|
||||
self._credential_cache: Dict[str, Tuple[str, Principal, float]] = {}
|
||||
self._cache_ttl = 60.0
|
||||
self._principal_cache: Dict[str, Tuple[Principal, float]] = {}
|
||||
self._secret_key_cache: Dict[str, Tuple[str, float]] = {}
|
||||
self._cache_ttl = float(os.environ.get("IAM_CACHE_TTL_SECONDS", "5.0"))
|
||||
self._last_stat_check = 0.0
|
||||
self._stat_check_interval = 1.0
|
||||
self._sessions: Dict[str, Dict[str, Any]] = {}
|
||||
self._session_lock = threading.Lock()
|
||||
self._load()
|
||||
self._load_lockout_state()
|
||||
|
||||
def _maybe_reload(self) -> None:
|
||||
"""Reload configuration if the file has changed on disk."""
|
||||
@@ -123,7 +140,8 @@ class IamService:
|
||||
try:
|
||||
if self.config_path.stat().st_mtime > self._last_load_time:
|
||||
self._load()
|
||||
self._credential_cache.clear()
|
||||
self._principal_cache.clear()
|
||||
self._secret_key_cache.clear()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
@@ -139,23 +157,64 @@ class IamService:
|
||||
f"Access temporarily locked. Try again in {seconds} seconds."
|
||||
)
|
||||
record = self._users.get(access_key)
|
||||
if not record or record["secret_key"] != secret_key:
|
||||
stored_secret = record["secret_key"] if record else secrets.token_urlsafe(24)
|
||||
if not record or not hmac.compare_digest(stored_secret, secret_key):
|
||||
self._record_failed_attempt(access_key)
|
||||
raise IamError("Invalid credentials")
|
||||
self._clear_failed_attempts(access_key)
|
||||
return self._build_principal(access_key, record)
|
||||
|
||||
_MAX_LOCKOUT_KEYS = 10000
|
||||
|
||||
def _record_failed_attempt(self, access_key: str) -> None:
|
||||
if not access_key:
|
||||
return
|
||||
if access_key not in self._failed_attempts and len(self._failed_attempts) >= self._MAX_LOCKOUT_KEYS:
|
||||
oldest_key = min(self._failed_attempts, key=lambda k: self._failed_attempts[k][0] if self._failed_attempts[k] else datetime.min.replace(tzinfo=timezone.utc))
|
||||
del self._failed_attempts[oldest_key]
|
||||
attempts = self._failed_attempts.setdefault(access_key, deque())
|
||||
self._prune_attempts(attempts)
|
||||
attempts.append(datetime.now(timezone.utc))
|
||||
self._save_lockout_state()
|
||||
|
||||
def _clear_failed_attempts(self, access_key: str) -> None:
|
||||
if not access_key:
|
||||
return
|
||||
self._failed_attempts.pop(access_key, None)
|
||||
if self._failed_attempts.pop(access_key, None) is not None:
|
||||
self._save_lockout_state()
|
||||
|
||||
def _lockout_file(self) -> Path:
|
||||
return self.config_path.parent / "lockout_state.json"
|
||||
|
||||
def _load_lockout_state(self) -> None:
|
||||
"""Load lockout state from disk."""
|
||||
try:
|
||||
if self._lockout_file().exists():
|
||||
data = json.loads(self._lockout_file().read_text(encoding="utf-8"))
|
||||
cutoff = datetime.now(timezone.utc) - self.auth_lockout_window
|
||||
for key, timestamps in data.get("failed_attempts", {}).items():
|
||||
valid = []
|
||||
for ts in timestamps:
|
||||
try:
|
||||
dt = datetime.fromisoformat(ts)
|
||||
if dt > cutoff:
|
||||
valid.append(dt)
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
if valid:
|
||||
self._failed_attempts[key] = deque(valid)
|
||||
except (OSError, json.JSONDecodeError):
|
||||
pass
|
||||
|
||||
def _save_lockout_state(self) -> None:
|
||||
"""Persist lockout state to disk."""
|
||||
data: Dict[str, Any] = {"failed_attempts": {}}
|
||||
for key, attempts in self._failed_attempts.items():
|
||||
data["failed_attempts"][key] = [ts.isoformat() for ts in attempts]
|
||||
try:
|
||||
self._lockout_file().write_text(json.dumps(data), encoding="utf-8")
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def _prune_attempts(self, attempts: Deque[datetime]) -> None:
|
||||
cutoff = datetime.now(timezone.utc) - self.auth_lockout_window
|
||||
@@ -198,16 +257,23 @@ class IamService:
|
||||
return token
|
||||
|
||||
def validate_session_token(self, access_key: str, session_token: str) -> bool:
|
||||
"""Validate a session token for an access key."""
|
||||
session = self._sessions.get(session_token)
|
||||
if not session:
|
||||
return False
|
||||
if session["access_key"] != access_key:
|
||||
return False
|
||||
if time.time() > session["expires_at"]:
|
||||
del self._sessions[session_token]
|
||||
return False
|
||||
return True
|
||||
"""Validate a session token for an access key (thread-safe, constant-time)."""
|
||||
dummy_key = secrets.token_urlsafe(16)
|
||||
dummy_token = secrets.token_urlsafe(32)
|
||||
with self._session_lock:
|
||||
session = self._sessions.get(session_token)
|
||||
if not session:
|
||||
hmac.compare_digest(access_key, dummy_key)
|
||||
hmac.compare_digest(session_token, dummy_token)
|
||||
return False
|
||||
key_match = hmac.compare_digest(session["access_key"], access_key)
|
||||
if not key_match:
|
||||
hmac.compare_digest(session_token, dummy_token)
|
||||
return False
|
||||
if time.time() > session["expires_at"]:
|
||||
self._sessions.pop(session_token, None)
|
||||
return False
|
||||
return True
|
||||
|
||||
def _cleanup_expired_sessions(self) -> None:
|
||||
"""Remove expired session tokens."""
|
||||
@@ -218,9 +284,9 @@ class IamService:
|
||||
|
||||
def principal_for_key(self, access_key: str) -> Principal:
|
||||
now = time.time()
|
||||
cached = self._credential_cache.get(access_key)
|
||||
cached = self._principal_cache.get(access_key)
|
||||
if cached:
|
||||
secret, principal, cached_time = cached
|
||||
principal, cached_time = cached
|
||||
if now - cached_time < self._cache_ttl:
|
||||
return principal
|
||||
|
||||
@@ -229,23 +295,14 @@ class IamService:
|
||||
if not record:
|
||||
raise IamError("Unknown access key")
|
||||
principal = self._build_principal(access_key, record)
|
||||
self._credential_cache[access_key] = (record["secret_key"], principal, now)
|
||||
self._principal_cache[access_key] = (principal, now)
|
||||
return principal
|
||||
|
||||
def secret_for_key(self, access_key: str) -> str:
|
||||
now = time.time()
|
||||
cached = self._credential_cache.get(access_key)
|
||||
if cached:
|
||||
secret, principal, cached_time = cached
|
||||
if now - cached_time < self._cache_ttl:
|
||||
return secret
|
||||
|
||||
self._maybe_reload()
|
||||
record = self._users.get(access_key)
|
||||
if not record:
|
||||
raise IamError("Unknown access key")
|
||||
principal = self._build_principal(access_key, record)
|
||||
self._credential_cache[access_key] = (record["secret_key"], principal, now)
|
||||
return record["secret_key"]
|
||||
|
||||
def authorize(self, principal: Principal, bucket_name: str | None, action: str) -> None:
|
||||
@@ -257,6 +314,18 @@ class IamService:
|
||||
if not self._is_allowed(principal, normalized, action):
|
||||
raise IamError(f"Access denied for action '{action}' on bucket '{bucket_name}'")
|
||||
|
||||
def check_permissions(self, principal: Principal, bucket_name: str | None, actions: Iterable[str]) -> Dict[str, bool]:
|
||||
self._maybe_reload()
|
||||
bucket_name = (bucket_name or "*").lower() if bucket_name != "*" else (bucket_name or "*")
|
||||
normalized_actions = {a: self._normalize_action(a) for a in actions}
|
||||
results: Dict[str, bool] = {}
|
||||
for original, canonical in normalized_actions.items():
|
||||
if canonical not in ALLOWED_ACTIONS:
|
||||
results[original] = False
|
||||
else:
|
||||
results[original] = self._is_allowed(principal, bucket_name, canonical)
|
||||
return results
|
||||
|
||||
def buckets_for_principal(self, principal: Principal, buckets: Iterable[str]) -> List[str]:
|
||||
return [bucket for bucket in buckets if self._is_allowed(principal, bucket, "list")]
|
||||
|
||||
@@ -317,6 +386,10 @@ class IamService:
|
||||
new_secret = self._generate_secret_key()
|
||||
user["secret_key"] = new_secret
|
||||
self._save()
|
||||
self._principal_cache.pop(access_key, None)
|
||||
self._secret_key_cache.pop(access_key, None)
|
||||
from .s3_api import clear_signing_key_cache
|
||||
clear_signing_key_cache()
|
||||
self._load()
|
||||
return new_secret
|
||||
|
||||
@@ -335,6 +408,10 @@ class IamService:
|
||||
raise IamError("User not found")
|
||||
self._raw_config["users"] = remaining
|
||||
self._save()
|
||||
self._principal_cache.pop(access_key, None)
|
||||
self._secret_key_cache.pop(access_key, None)
|
||||
from .s3_api import clear_signing_key_cache
|
||||
clear_signing_key_cache()
|
||||
self._load()
|
||||
|
||||
def update_user_policies(self, access_key: str, policies: Sequence[Dict[str, Any]]) -> None:
|
||||
@@ -469,11 +546,13 @@ class IamService:
|
||||
return candidate if candidate in ALLOWED_ACTIONS else ""
|
||||
|
||||
def _write_default(self) -> None:
|
||||
access_key = secrets.token_hex(12)
|
||||
secret_key = secrets.token_urlsafe(32)
|
||||
default = {
|
||||
"users": [
|
||||
{
|
||||
"access_key": "localadmin",
|
||||
"secret_key": "localadmin",
|
||||
"access_key": access_key,
|
||||
"secret_key": secret_key,
|
||||
"display_name": "Local Admin",
|
||||
"policies": [
|
||||
{"bucket": "*", "actions": list(ALLOWED_ACTIONS)}
|
||||
@@ -482,6 +561,14 @@ class IamService:
|
||||
]
|
||||
}
|
||||
self.config_path.write_text(json.dumps(default, indent=2))
|
||||
print(f"\n{'='*60}")
|
||||
print("MYFSIO FIRST RUN - ADMIN CREDENTIALS GENERATED")
|
||||
print(f"{'='*60}")
|
||||
print(f"Access Key: {access_key}")
|
||||
print(f"Secret Key: {secret_key}")
|
||||
print(f"{'='*60}")
|
||||
print(f"Missed this? Check: {self.config_path}")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
def _generate_access_key(self) -> str:
|
||||
return secrets.token_hex(8)
|
||||
@@ -497,25 +584,25 @@ class IamService:
|
||||
|
||||
def get_secret_key(self, access_key: str) -> str | None:
|
||||
now = time.time()
|
||||
cached = self._credential_cache.get(access_key)
|
||||
cached = self._secret_key_cache.get(access_key)
|
||||
if cached:
|
||||
secret, principal, cached_time = cached
|
||||
secret_key, cached_time = cached
|
||||
if now - cached_time < self._cache_ttl:
|
||||
return secret
|
||||
return secret_key
|
||||
|
||||
self._maybe_reload()
|
||||
record = self._users.get(access_key)
|
||||
if record:
|
||||
principal = self._build_principal(access_key, record)
|
||||
self._credential_cache[access_key] = (record["secret_key"], principal, now)
|
||||
return record["secret_key"]
|
||||
secret_key = record["secret_key"]
|
||||
self._secret_key_cache[access_key] = (secret_key, now)
|
||||
return secret_key
|
||||
return None
|
||||
|
||||
def get_principal(self, access_key: str) -> Principal | None:
|
||||
now = time.time()
|
||||
cached = self._credential_cache.get(access_key)
|
||||
cached = self._principal_cache.get(access_key)
|
||||
if cached:
|
||||
secret, principal, cached_time = cached
|
||||
principal, cached_time = cached
|
||||
if now - cached_time < self._cache_ttl:
|
||||
return principal
|
||||
|
||||
@@ -523,6 +610,6 @@ class IamService:
|
||||
record = self._users.get(access_key)
|
||||
if record:
|
||||
principal = self._build_principal(access_key, record)
|
||||
self._credential_cache[access_key] = (record["secret_key"], principal, now)
|
||||
self._principal_cache[access_key] = (principal, now)
|
||||
return principal
|
||||
return None
|
||||
|
||||
169
app/kms.py
169
app/kms.py
@@ -2,7 +2,11 @@ from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import secrets
|
||||
import subprocess
|
||||
import sys
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
@@ -13,6 +17,30 @@ from cryptography.hazmat.primitives.ciphers.aead import AESGCM
|
||||
|
||||
from .encryption import EncryptionError, EncryptionProvider, EncryptionResult
|
||||
|
||||
if sys.platform != "win32":
|
||||
import fcntl
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _set_secure_file_permissions(file_path: Path) -> None:
|
||||
"""Set restrictive file permissions (owner read/write only)."""
|
||||
if sys.platform == "win32":
|
||||
try:
|
||||
username = os.environ.get("USERNAME", "")
|
||||
if username:
|
||||
subprocess.run(
|
||||
["icacls", str(file_path), "/inheritance:r",
|
||||
"/grant:r", f"{username}:F"],
|
||||
check=True, capture_output=True
|
||||
)
|
||||
else:
|
||||
logger.warning("Could not set secure permissions on %s: USERNAME not set", file_path)
|
||||
except (subprocess.SubprocessError, OSError) as exc:
|
||||
logger.warning("Failed to set secure permissions on %s: %s", file_path, exc)
|
||||
else:
|
||||
os.chmod(file_path, 0o600)
|
||||
|
||||
|
||||
@dataclass
|
||||
class KMSKey:
|
||||
@@ -74,11 +102,11 @@ class KMSEncryptionProvider(EncryptionProvider):
|
||||
def encrypt(self, plaintext: bytes, context: Dict[str, str] | None = None) -> EncryptionResult:
|
||||
"""Encrypt data using envelope encryption with KMS."""
|
||||
data_key, encrypted_data_key = self.generate_data_key()
|
||||
|
||||
|
||||
aesgcm = AESGCM(data_key)
|
||||
nonce = secrets.token_bytes(12)
|
||||
ciphertext = aesgcm.encrypt(nonce, plaintext,
|
||||
json.dumps(context).encode() if context else None)
|
||||
ciphertext = aesgcm.encrypt(nonce, plaintext,
|
||||
json.dumps(context, sort_keys=True).encode() if context else None)
|
||||
|
||||
return EncryptionResult(
|
||||
ciphertext=ciphertext,
|
||||
@@ -90,15 +118,26 @@ class KMSEncryptionProvider(EncryptionProvider):
|
||||
def decrypt(self, ciphertext: bytes, nonce: bytes, encrypted_data_key: bytes,
|
||||
key_id: str, context: Dict[str, str] | None = None) -> bytes:
|
||||
"""Decrypt data using envelope encryption with KMS."""
|
||||
# Note: Data key is encrypted without context (AAD), so we decrypt without context
|
||||
data_key = self.kms.decrypt_data_key(key_id, encrypted_data_key, context=None)
|
||||
|
||||
if len(data_key) != 32:
|
||||
raise EncryptionError("Invalid data key size")
|
||||
|
||||
aesgcm = AESGCM(data_key)
|
||||
try:
|
||||
return aesgcm.decrypt(nonce, ciphertext,
|
||||
json.dumps(context).encode() if context else None)
|
||||
json.dumps(context, sort_keys=True).encode() if context else None)
|
||||
except Exception as exc:
|
||||
raise EncryptionError(f"Failed to decrypt data: {exc}") from exc
|
||||
logger.debug("KMS decryption failed: %s", exc)
|
||||
raise EncryptionError("Failed to decrypt data") from exc
|
||||
|
||||
def decrypt_data_key(self, encrypted_data_key: bytes, key_id: str | None = None) -> bytes:
|
||||
"""Decrypt an encrypted data key using KMS."""
|
||||
if key_id is None:
|
||||
key_id = self.key_id
|
||||
data_key = self.kms.decrypt_data_key(key_id, encrypted_data_key, context=None)
|
||||
if len(data_key) != 32:
|
||||
raise EncryptionError("Invalid data key size")
|
||||
return data_key
|
||||
|
||||
|
||||
class KMSManager:
|
||||
@@ -108,27 +147,52 @@ class KMSManager:
|
||||
Keys are stored encrypted on disk.
|
||||
"""
|
||||
|
||||
def __init__(self, keys_path: Path, master_key_path: Path):
|
||||
def __init__(
|
||||
self,
|
||||
keys_path: Path,
|
||||
master_key_path: Path,
|
||||
generate_data_key_min_bytes: int = 1,
|
||||
generate_data_key_max_bytes: int = 1024,
|
||||
):
|
||||
self.keys_path = keys_path
|
||||
self.master_key_path = master_key_path
|
||||
self.generate_data_key_min_bytes = generate_data_key_min_bytes
|
||||
self.generate_data_key_max_bytes = generate_data_key_max_bytes
|
||||
self._keys: Dict[str, KMSKey] = {}
|
||||
self._master_key: bytes | None = None
|
||||
self._master_aesgcm: AESGCM | None = None
|
||||
self._loaded = False
|
||||
|
||||
@property
|
||||
def master_key(self) -> bytes:
|
||||
"""Load or create the master key for encrypting KMS keys."""
|
||||
"""Load or create the master key for encrypting KMS keys (with file locking)."""
|
||||
if self._master_key is None:
|
||||
if self.master_key_path.exists():
|
||||
self._master_key = base64.b64decode(
|
||||
self.master_key_path.read_text().strip()
|
||||
)
|
||||
else:
|
||||
self._master_key = secrets.token_bytes(32)
|
||||
self.master_key_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self.master_key_path.write_text(
|
||||
base64.b64encode(self._master_key).decode()
|
||||
)
|
||||
lock_path = self.master_key_path.with_suffix(".lock")
|
||||
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(lock_path, "w") as lock_file:
|
||||
if sys.platform == "win32":
|
||||
import msvcrt
|
||||
msvcrt.locking(lock_file.fileno(), msvcrt.LK_LOCK, 1)
|
||||
else:
|
||||
fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX)
|
||||
try:
|
||||
if self.master_key_path.exists():
|
||||
self._master_key = base64.b64decode(
|
||||
self.master_key_path.read_text().strip()
|
||||
)
|
||||
else:
|
||||
self._master_key = secrets.token_bytes(32)
|
||||
self.master_key_path.write_text(
|
||||
base64.b64encode(self._master_key).decode()
|
||||
)
|
||||
_set_secure_file_permissions(self.master_key_path)
|
||||
finally:
|
||||
if sys.platform == "win32":
|
||||
import msvcrt
|
||||
msvcrt.locking(lock_file.fileno(), msvcrt.LK_UNLCK, 1)
|
||||
else:
|
||||
fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
|
||||
self._master_aesgcm = AESGCM(self._master_key)
|
||||
return self._master_key
|
||||
|
||||
def _load_keys(self) -> None:
|
||||
@@ -145,8 +209,10 @@ class KMSManager:
|
||||
encrypted = base64.b64decode(key_data["EncryptedKeyMaterial"])
|
||||
key.key_material = self._decrypt_key_material(encrypted)
|
||||
self._keys[key.key_id] = key
|
||||
except Exception:
|
||||
pass
|
||||
except json.JSONDecodeError as exc:
|
||||
logger.error("Failed to parse KMS keys file: %s", exc)
|
||||
except (ValueError, KeyError) as exc:
|
||||
logger.error("Invalid KMS key data: %s", exc)
|
||||
|
||||
self._loaded = True
|
||||
|
||||
@@ -158,26 +224,25 @@ class KMSManager:
|
||||
encrypted = self._encrypt_key_material(key.key_material)
|
||||
data["EncryptedKeyMaterial"] = base64.b64encode(encrypted).decode()
|
||||
keys_data.append(data)
|
||||
|
||||
|
||||
self.keys_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self.keys_path.write_text(
|
||||
json.dumps({"keys": keys_data}, indent=2),
|
||||
encoding="utf-8"
|
||||
)
|
||||
_set_secure_file_permissions(self.keys_path)
|
||||
|
||||
def _encrypt_key_material(self, key_material: bytes) -> bytes:
|
||||
"""Encrypt key material with the master key."""
|
||||
aesgcm = AESGCM(self.master_key)
|
||||
_ = self.master_key
|
||||
nonce = secrets.token_bytes(12)
|
||||
ciphertext = aesgcm.encrypt(nonce, key_material, None)
|
||||
ciphertext = self._master_aesgcm.encrypt(nonce, key_material, None)
|
||||
return nonce + ciphertext
|
||||
|
||||
|
||||
def _decrypt_key_material(self, encrypted: bytes) -> bytes:
|
||||
"""Decrypt key material with the master key."""
|
||||
aesgcm = AESGCM(self.master_key)
|
||||
_ = self.master_key
|
||||
nonce = encrypted[:12]
|
||||
ciphertext = encrypted[12:]
|
||||
return aesgcm.decrypt(nonce, ciphertext, None)
|
||||
return self._master_aesgcm.decrypt(nonce, ciphertext, None)
|
||||
|
||||
def create_key(self, description: str = "", key_id: str | None = None) -> KMSKey:
|
||||
"""Create a new KMS key."""
|
||||
@@ -269,7 +334,7 @@ class KMSManager:
|
||||
|
||||
aesgcm = AESGCM(key.key_material)
|
||||
nonce = secrets.token_bytes(12)
|
||||
aad = json.dumps(context).encode() if context else None
|
||||
aad = json.dumps(context, sort_keys=True).encode() if context else None
|
||||
ciphertext = aesgcm.encrypt(nonce, plaintext, aad)
|
||||
|
||||
key_id_bytes = key_id.encode("utf-8")
|
||||
@@ -298,17 +363,24 @@ class KMSManager:
|
||||
encrypted = rest[12:]
|
||||
|
||||
aesgcm = AESGCM(key.key_material)
|
||||
aad = json.dumps(context).encode() if context else None
|
||||
aad = json.dumps(context, sort_keys=True).encode() if context else None
|
||||
try:
|
||||
plaintext = aesgcm.decrypt(nonce, encrypted, aad)
|
||||
return plaintext, key_id
|
||||
except Exception as exc:
|
||||
raise EncryptionError(f"Decryption failed: {exc}") from exc
|
||||
logger.debug("KMS decrypt operation failed: %s", exc)
|
||||
raise EncryptionError("Decryption failed") from exc
|
||||
|
||||
def generate_data_key(self, key_id: str,
|
||||
context: Dict[str, str] | None = None) -> tuple[bytes, bytes]:
|
||||
context: Dict[str, str] | None = None,
|
||||
key_spec: str = "AES_256") -> tuple[bytes, bytes]:
|
||||
"""Generate a data key and return both plaintext and encrypted versions.
|
||||
|
||||
|
||||
Args:
|
||||
key_id: The KMS key ID to use for encryption
|
||||
context: Optional encryption context
|
||||
key_spec: Key specification - AES_128 or AES_256 (default)
|
||||
|
||||
Returns:
|
||||
Tuple of (plaintext_key, encrypted_key)
|
||||
"""
|
||||
@@ -318,11 +390,12 @@ class KMSManager:
|
||||
raise EncryptionError(f"Key not found: {key_id}")
|
||||
if not key.enabled:
|
||||
raise EncryptionError(f"Key is disabled: {key_id}")
|
||||
|
||||
plaintext_key = secrets.token_bytes(32)
|
||||
|
||||
key_bytes = 32 if key_spec == "AES_256" else 16
|
||||
plaintext_key = secrets.token_bytes(key_bytes)
|
||||
|
||||
encrypted_key = self.encrypt(key_id, plaintext_key, context)
|
||||
|
||||
|
||||
return plaintext_key, encrypted_key
|
||||
|
||||
def decrypt_data_key(self, key_id: str, encrypted_key: bytes,
|
||||
@@ -331,22 +404,6 @@ class KMSManager:
|
||||
plaintext, _ = self.decrypt(encrypted_key, context)
|
||||
return plaintext
|
||||
|
||||
def get_provider(self, key_id: str | None = None) -> KMSEncryptionProvider:
|
||||
"""Get an encryption provider for a specific key."""
|
||||
self._load_keys()
|
||||
|
||||
if key_id is None:
|
||||
if not self._keys:
|
||||
key = self.create_key("Default KMS Key")
|
||||
key_id = key.key_id
|
||||
else:
|
||||
key_id = next(iter(self._keys.keys()))
|
||||
|
||||
if key_id not in self._keys:
|
||||
raise EncryptionError(f"Key not found: {key_id}")
|
||||
|
||||
return KMSEncryptionProvider(self, key_id)
|
||||
|
||||
def re_encrypt(self, ciphertext: bytes, destination_key_id: str,
|
||||
source_context: Dict[str, str] | None = None,
|
||||
destination_context: Dict[str, str] | None = None) -> bytes:
|
||||
@@ -358,6 +415,8 @@ class KMSManager:
|
||||
|
||||
def generate_random(self, num_bytes: int = 32) -> bytes:
|
||||
"""Generate cryptographically secure random bytes."""
|
||||
if num_bytes < 1 or num_bytes > 1024:
|
||||
raise EncryptionError("Number of bytes must be between 1 and 1024")
|
||||
if num_bytes < self.generate_data_key_min_bytes or num_bytes > self.generate_data_key_max_bytes:
|
||||
raise EncryptionError(
|
||||
f"Number of bytes must be between {self.generate_data_key_min_bytes} and {self.generate_data_key_max_bytes}"
|
||||
)
|
||||
return secrets.token_bytes(num_bytes)
|
||||
|
||||
@@ -71,10 +71,9 @@ class LifecycleExecutionRecord:
|
||||
|
||||
|
||||
class LifecycleHistoryStore:
|
||||
MAX_HISTORY_PER_BUCKET = 50
|
||||
|
||||
def __init__(self, storage_root: Path) -> None:
|
||||
def __init__(self, storage_root: Path, max_history_per_bucket: int = 50) -> None:
|
||||
self.storage_root = storage_root
|
||||
self.max_history_per_bucket = max_history_per_bucket
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def _get_history_path(self, bucket_name: str) -> Path:
|
||||
@@ -95,7 +94,7 @@ class LifecycleHistoryStore:
|
||||
def save_history(self, bucket_name: str, records: List[LifecycleExecutionRecord]) -> None:
|
||||
path = self._get_history_path(bucket_name)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
data = {"executions": [r.to_dict() for r in records[:self.MAX_HISTORY_PER_BUCKET]]}
|
||||
data = {"executions": [r.to_dict() for r in records[:self.max_history_per_bucket]]}
|
||||
try:
|
||||
with open(path, "w") as f:
|
||||
json.dump(data, f, indent=2)
|
||||
@@ -114,14 +113,20 @@ class LifecycleHistoryStore:
|
||||
|
||||
|
||||
class LifecycleManager:
|
||||
def __init__(self, storage: ObjectStorage, interval_seconds: int = 3600, storage_root: Optional[Path] = None):
|
||||
def __init__(
|
||||
self,
|
||||
storage: ObjectStorage,
|
||||
interval_seconds: int = 3600,
|
||||
storage_root: Optional[Path] = None,
|
||||
max_history_per_bucket: int = 50,
|
||||
):
|
||||
self.storage = storage
|
||||
self.interval_seconds = interval_seconds
|
||||
self.storage_root = storage_root
|
||||
self._timer: Optional[threading.Timer] = None
|
||||
self._shutdown = False
|
||||
self._lock = threading.Lock()
|
||||
self.history_store = LifecycleHistoryStore(storage_root) if storage_root else None
|
||||
self.history_store = LifecycleHistoryStore(storage_root, max_history_per_bucket) if storage_root else None
|
||||
|
||||
def start(self) -> None:
|
||||
if self._timer is not None:
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import ipaddress
|
||||
import json
|
||||
import logging
|
||||
import queue
|
||||
import socket
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
@@ -13,6 +15,71 @@ from typing import Any, Dict, List, Optional
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import requests
|
||||
from urllib3.util.connection import create_connection as _urllib3_create_connection
|
||||
|
||||
|
||||
def _resolve_and_check_url(url: str, allow_internal: bool = False) -> Optional[str]:
|
||||
try:
|
||||
parsed = urlparse(url)
|
||||
hostname = parsed.hostname
|
||||
if not hostname:
|
||||
return None
|
||||
cloud_metadata_hosts = {
|
||||
"metadata.google.internal",
|
||||
"169.254.169.254",
|
||||
}
|
||||
if hostname.lower() in cloud_metadata_hosts:
|
||||
return None
|
||||
if allow_internal:
|
||||
return hostname
|
||||
blocked_hosts = {
|
||||
"localhost",
|
||||
"127.0.0.1",
|
||||
"0.0.0.0",
|
||||
"::1",
|
||||
"[::1]",
|
||||
}
|
||||
if hostname.lower() in blocked_hosts:
|
||||
return None
|
||||
try:
|
||||
resolved_ip = socket.gethostbyname(hostname)
|
||||
ip = ipaddress.ip_address(resolved_ip)
|
||||
if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved:
|
||||
return None
|
||||
return resolved_ip
|
||||
except (socket.gaierror, ValueError):
|
||||
return None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _is_safe_url(url: str, allow_internal: bool = False) -> bool:
|
||||
return _resolve_and_check_url(url, allow_internal) is not None
|
||||
|
||||
|
||||
_dns_pin_lock = threading.Lock()
|
||||
|
||||
|
||||
def _pinned_post(url: str, pinned_ip: str, **kwargs: Any) -> requests.Response:
|
||||
parsed = urlparse(url)
|
||||
hostname = parsed.hostname or ""
|
||||
session = requests.Session()
|
||||
original_create = _urllib3_create_connection
|
||||
|
||||
def _create_pinned(address: Any, *args: Any, **kw: Any) -> Any:
|
||||
host, req_port = address
|
||||
if host == hostname:
|
||||
return original_create((pinned_ip, req_port), *args, **kw)
|
||||
return original_create(address, *args, **kw)
|
||||
|
||||
import urllib3.util.connection as _conn_mod
|
||||
with _dns_pin_lock:
|
||||
_conn_mod.create_connection = _create_pinned
|
||||
try:
|
||||
return session.post(url, **kwargs)
|
||||
finally:
|
||||
_conn_mod.create_connection = original_create
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -165,8 +232,9 @@ class NotificationConfiguration:
|
||||
|
||||
|
||||
class NotificationService:
|
||||
def __init__(self, storage_root: Path, worker_count: int = 2):
|
||||
def __init__(self, storage_root: Path, worker_count: int = 2, allow_internal_endpoints: bool = False):
|
||||
self.storage_root = storage_root
|
||||
self._allow_internal_endpoints = allow_internal_endpoints
|
||||
self._configs: Dict[str, List[NotificationConfiguration]] = {}
|
||||
self._queue: queue.Queue[tuple[NotificationEvent, WebhookDestination]] = queue.Queue()
|
||||
self._workers: List[threading.Thread] = []
|
||||
@@ -299,14 +367,18 @@ class NotificationService:
|
||||
self._queue.task_done()
|
||||
|
||||
def _send_notification(self, event: NotificationEvent, destination: WebhookDestination) -> None:
|
||||
resolved_ip = _resolve_and_check_url(destination.url, allow_internal=self._allow_internal_endpoints)
|
||||
if not resolved_ip:
|
||||
raise RuntimeError(f"Blocked request (SSRF protection): {destination.url}")
|
||||
payload = event.to_s3_event()
|
||||
headers = {"Content-Type": "application/json", **destination.headers}
|
||||
|
||||
last_error = None
|
||||
for attempt in range(destination.retry_count):
|
||||
try:
|
||||
response = requests.post(
|
||||
response = _pinned_post(
|
||||
destination.url,
|
||||
resolved_ip,
|
||||
json=payload,
|
||||
headers=headers,
|
||||
timeout=destination.timeout_seconds,
|
||||
|
||||
300
app/operation_metrics.py
Normal file
300
app/operation_metrics.py
Normal file
@@ -0,0 +1,300 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import random
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
MAX_LATENCY_SAMPLES = 5000
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class OperationStats:
|
||||
count: int = 0
|
||||
success_count: int = 0
|
||||
error_count: int = 0
|
||||
latency_sum_ms: float = 0.0
|
||||
latency_min_ms: float = float("inf")
|
||||
latency_max_ms: float = 0.0
|
||||
bytes_in: int = 0
|
||||
bytes_out: int = 0
|
||||
latency_samples: List[float] = field(default_factory=list)
|
||||
|
||||
@staticmethod
|
||||
def _compute_percentile(sorted_data: List[float], p: float) -> float:
|
||||
if not sorted_data:
|
||||
return 0.0
|
||||
k = (len(sorted_data) - 1) * (p / 100.0)
|
||||
f = int(k)
|
||||
c = min(f + 1, len(sorted_data) - 1)
|
||||
d = k - f
|
||||
return sorted_data[f] + d * (sorted_data[c] - sorted_data[f])
|
||||
|
||||
def record(self, latency_ms: float, success: bool, bytes_in: int = 0, bytes_out: int = 0) -> None:
|
||||
self.count += 1
|
||||
if success:
|
||||
self.success_count += 1
|
||||
else:
|
||||
self.error_count += 1
|
||||
self.latency_sum_ms += latency_ms
|
||||
if latency_ms < self.latency_min_ms:
|
||||
self.latency_min_ms = latency_ms
|
||||
if latency_ms > self.latency_max_ms:
|
||||
self.latency_max_ms = latency_ms
|
||||
self.bytes_in += bytes_in
|
||||
self.bytes_out += bytes_out
|
||||
if len(self.latency_samples) < MAX_LATENCY_SAMPLES:
|
||||
self.latency_samples.append(latency_ms)
|
||||
else:
|
||||
j = random.randint(0, self.count - 1)
|
||||
if j < MAX_LATENCY_SAMPLES:
|
||||
self.latency_samples[j] = latency_ms
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
avg_latency = self.latency_sum_ms / self.count if self.count > 0 else 0.0
|
||||
min_latency = self.latency_min_ms if self.latency_min_ms != float("inf") else 0.0
|
||||
sorted_latencies = sorted(self.latency_samples)
|
||||
return {
|
||||
"count": self.count,
|
||||
"success_count": self.success_count,
|
||||
"error_count": self.error_count,
|
||||
"latency_avg_ms": round(avg_latency, 2),
|
||||
"latency_min_ms": round(min_latency, 2),
|
||||
"latency_max_ms": round(self.latency_max_ms, 2),
|
||||
"latency_p50_ms": round(self._compute_percentile(sorted_latencies, 50), 2),
|
||||
"latency_p95_ms": round(self._compute_percentile(sorted_latencies, 95), 2),
|
||||
"latency_p99_ms": round(self._compute_percentile(sorted_latencies, 99), 2),
|
||||
"bytes_in": self.bytes_in,
|
||||
"bytes_out": self.bytes_out,
|
||||
}
|
||||
|
||||
def merge(self, other: "OperationStats") -> None:
|
||||
self.count += other.count
|
||||
self.success_count += other.success_count
|
||||
self.error_count += other.error_count
|
||||
self.latency_sum_ms += other.latency_sum_ms
|
||||
if other.latency_min_ms < self.latency_min_ms:
|
||||
self.latency_min_ms = other.latency_min_ms
|
||||
if other.latency_max_ms > self.latency_max_ms:
|
||||
self.latency_max_ms = other.latency_max_ms
|
||||
self.bytes_in += other.bytes_in
|
||||
self.bytes_out += other.bytes_out
|
||||
combined = self.latency_samples + other.latency_samples
|
||||
if len(combined) > MAX_LATENCY_SAMPLES:
|
||||
random.shuffle(combined)
|
||||
combined = combined[:MAX_LATENCY_SAMPLES]
|
||||
self.latency_samples = combined
|
||||
|
||||
|
||||
@dataclass
|
||||
class MetricsSnapshot:
|
||||
timestamp: datetime
|
||||
window_seconds: int
|
||||
by_method: Dict[str, Dict[str, Any]]
|
||||
by_endpoint: Dict[str, Dict[str, Any]]
|
||||
by_status_class: Dict[str, int]
|
||||
error_codes: Dict[str, int]
|
||||
totals: Dict[str, Any]
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"timestamp": self.timestamp.isoformat(),
|
||||
"window_seconds": self.window_seconds,
|
||||
"by_method": self.by_method,
|
||||
"by_endpoint": self.by_endpoint,
|
||||
"by_status_class": self.by_status_class,
|
||||
"error_codes": self.error_codes,
|
||||
"totals": self.totals,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "MetricsSnapshot":
|
||||
return cls(
|
||||
timestamp=datetime.fromisoformat(data["timestamp"]),
|
||||
window_seconds=data.get("window_seconds", 300),
|
||||
by_method=data.get("by_method", {}),
|
||||
by_endpoint=data.get("by_endpoint", {}),
|
||||
by_status_class=data.get("by_status_class", {}),
|
||||
error_codes=data.get("error_codes", {}),
|
||||
totals=data.get("totals", {}),
|
||||
)
|
||||
|
||||
|
||||
class OperationMetricsCollector:
|
||||
def __init__(
|
||||
self,
|
||||
storage_root: Path,
|
||||
interval_minutes: int = 5,
|
||||
retention_hours: int = 24,
|
||||
):
|
||||
self.storage_root = storage_root
|
||||
self.interval_seconds = interval_minutes * 60
|
||||
self.retention_hours = retention_hours
|
||||
self._lock = threading.Lock()
|
||||
self._by_method: Dict[str, OperationStats] = {}
|
||||
self._by_endpoint: Dict[str, OperationStats] = {}
|
||||
self._by_status_class: Dict[str, int] = {}
|
||||
self._error_codes: Dict[str, int] = {}
|
||||
self._totals = OperationStats()
|
||||
self._window_start = time.time()
|
||||
self._shutdown = threading.Event()
|
||||
self._snapshots: List[MetricsSnapshot] = []
|
||||
|
||||
self._load_history()
|
||||
|
||||
self._snapshot_thread = threading.Thread(
|
||||
target=self._snapshot_loop, name="operation-metrics-snapshot", daemon=True
|
||||
)
|
||||
self._snapshot_thread.start()
|
||||
|
||||
def _config_path(self) -> Path:
|
||||
return self.storage_root / ".myfsio.sys" / "config" / "operation_metrics.json"
|
||||
|
||||
def _load_history(self) -> None:
|
||||
config_path = self._config_path()
|
||||
if not config_path.exists():
|
||||
return
|
||||
try:
|
||||
data = json.loads(config_path.read_text(encoding="utf-8"))
|
||||
snapshots_data = data.get("snapshots", [])
|
||||
self._snapshots = [MetricsSnapshot.from_dict(s) for s in snapshots_data]
|
||||
self._prune_old_snapshots()
|
||||
except (json.JSONDecodeError, OSError, KeyError) as e:
|
||||
logger.warning(f"Failed to load operation metrics history: {e}")
|
||||
|
||||
def _save_history(self) -> None:
|
||||
config_path = self._config_path()
|
||||
config_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
data = {"snapshots": [s.to_dict() for s in self._snapshots]}
|
||||
config_path.write_text(json.dumps(data, indent=2), encoding="utf-8")
|
||||
except OSError as e:
|
||||
logger.warning(f"Failed to save operation metrics history: {e}")
|
||||
|
||||
def _prune_old_snapshots(self) -> None:
|
||||
if not self._snapshots:
|
||||
return
|
||||
cutoff = datetime.now(timezone.utc).timestamp() - (self.retention_hours * 3600)
|
||||
self._snapshots = [
|
||||
s for s in self._snapshots if s.timestamp.timestamp() > cutoff
|
||||
]
|
||||
|
||||
def _snapshot_loop(self) -> None:
|
||||
while not self._shutdown.is_set():
|
||||
self._shutdown.wait(timeout=self.interval_seconds)
|
||||
if not self._shutdown.is_set():
|
||||
self._take_snapshot()
|
||||
|
||||
def _take_snapshot(self) -> None:
|
||||
with self._lock:
|
||||
now = datetime.now(timezone.utc)
|
||||
window_seconds = int(time.time() - self._window_start)
|
||||
|
||||
snapshot = MetricsSnapshot(
|
||||
timestamp=now,
|
||||
window_seconds=window_seconds,
|
||||
by_method={k: v.to_dict() for k, v in self._by_method.items()},
|
||||
by_endpoint={k: v.to_dict() for k, v in self._by_endpoint.items()},
|
||||
by_status_class=dict(self._by_status_class),
|
||||
error_codes=dict(self._error_codes),
|
||||
totals=self._totals.to_dict(),
|
||||
)
|
||||
|
||||
self._snapshots.append(snapshot)
|
||||
self._prune_old_snapshots()
|
||||
self._save_history()
|
||||
|
||||
self._by_method.clear()
|
||||
self._by_endpoint.clear()
|
||||
self._by_status_class.clear()
|
||||
self._error_codes.clear()
|
||||
self._totals = OperationStats()
|
||||
self._window_start = time.time()
|
||||
|
||||
def record_request(
|
||||
self,
|
||||
method: str,
|
||||
endpoint_type: str,
|
||||
status_code: int,
|
||||
latency_ms: float,
|
||||
bytes_in: int = 0,
|
||||
bytes_out: int = 0,
|
||||
error_code: Optional[str] = None,
|
||||
) -> None:
|
||||
success = 200 <= status_code < 400
|
||||
status_class = f"{status_code // 100}xx"
|
||||
|
||||
with self._lock:
|
||||
if method not in self._by_method:
|
||||
self._by_method[method] = OperationStats()
|
||||
self._by_method[method].record(latency_ms, success, bytes_in, bytes_out)
|
||||
|
||||
if endpoint_type not in self._by_endpoint:
|
||||
self._by_endpoint[endpoint_type] = OperationStats()
|
||||
self._by_endpoint[endpoint_type].record(latency_ms, success, bytes_in, bytes_out)
|
||||
|
||||
self._by_status_class[status_class] = self._by_status_class.get(status_class, 0) + 1
|
||||
|
||||
if error_code:
|
||||
self._error_codes[error_code] = self._error_codes.get(error_code, 0) + 1
|
||||
|
||||
self._totals.record(latency_ms, success, bytes_in, bytes_out)
|
||||
|
||||
def get_current_stats(self) -> Dict[str, Any]:
|
||||
with self._lock:
|
||||
window_seconds = int(time.time() - self._window_start)
|
||||
return {
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"window_seconds": window_seconds,
|
||||
"by_method": {k: v.to_dict() for k, v in self._by_method.items()},
|
||||
"by_endpoint": {k: v.to_dict() for k, v in self._by_endpoint.items()},
|
||||
"by_status_class": dict(self._by_status_class),
|
||||
"error_codes": dict(self._error_codes),
|
||||
"totals": self._totals.to_dict(),
|
||||
}
|
||||
|
||||
def get_history(self, hours: Optional[int] = None) -> List[Dict[str, Any]]:
|
||||
with self._lock:
|
||||
snapshots = list(self._snapshots)
|
||||
|
||||
if hours:
|
||||
cutoff = datetime.now(timezone.utc).timestamp() - (hours * 3600)
|
||||
snapshots = [s for s in snapshots if s.timestamp.timestamp() > cutoff]
|
||||
|
||||
return [s.to_dict() for s in snapshots]
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self._shutdown.set()
|
||||
self._take_snapshot()
|
||||
self._snapshot_thread.join(timeout=5.0)
|
||||
|
||||
|
||||
def classify_endpoint(path: str) -> str:
|
||||
if not path or path == "/":
|
||||
return "service"
|
||||
|
||||
path = path.rstrip("/")
|
||||
|
||||
if path.startswith("/ui"):
|
||||
return "ui"
|
||||
|
||||
if path.startswith("/kms"):
|
||||
return "kms"
|
||||
|
||||
if path.startswith("/myfsio"):
|
||||
return "service"
|
||||
|
||||
parts = path.lstrip("/").split("/")
|
||||
if len(parts) == 0:
|
||||
return "service"
|
||||
elif len(parts) == 1:
|
||||
return "bucket"
|
||||
else:
|
||||
return "object"
|
||||
@@ -21,15 +21,20 @@ from .storage import ObjectStorage, StorageError
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
REPLICATION_USER_AGENT = "S3ReplicationAgent/1.0"
|
||||
REPLICATION_CONNECT_TIMEOUT = 5
|
||||
REPLICATION_READ_TIMEOUT = 30
|
||||
STREAMING_THRESHOLD_BYTES = 10 * 1024 * 1024
|
||||
|
||||
REPLICATION_MODE_NEW_ONLY = "new_only"
|
||||
REPLICATION_MODE_ALL = "all"
|
||||
REPLICATION_MODE_BIDIRECTIONAL = "bidirectional"
|
||||
|
||||
|
||||
def _create_s3_client(connection: RemoteConnection, *, health_check: bool = False) -> Any:
|
||||
def _create_s3_client(
|
||||
connection: RemoteConnection,
|
||||
*,
|
||||
health_check: bool = False,
|
||||
connect_timeout: int = 5,
|
||||
read_timeout: int = 30,
|
||||
max_retries: int = 2,
|
||||
) -> Any:
|
||||
"""Create a boto3 S3 client for the given connection.
|
||||
Args:
|
||||
connection: Remote S3 connection configuration
|
||||
@@ -37,9 +42,9 @@ def _create_s3_client(connection: RemoteConnection, *, health_check: bool = Fals
|
||||
"""
|
||||
config = Config(
|
||||
user_agent_extra=REPLICATION_USER_AGENT,
|
||||
connect_timeout=REPLICATION_CONNECT_TIMEOUT,
|
||||
read_timeout=REPLICATION_READ_TIMEOUT,
|
||||
retries={'max_attempts': 1 if health_check else 2},
|
||||
connect_timeout=connect_timeout,
|
||||
read_timeout=read_timeout,
|
||||
retries={'max_attempts': 1 if health_check else max_retries},
|
||||
signature_version='s3v4',
|
||||
s3={'addressing_style': 'path'},
|
||||
request_checksum_calculation='when_required',
|
||||
@@ -127,10 +132,13 @@ class ReplicationRule:
|
||||
target_connection_id: str
|
||||
target_bucket: str
|
||||
enabled: bool = True
|
||||
mode: str = REPLICATION_MODE_NEW_ONLY
|
||||
mode: str = REPLICATION_MODE_NEW_ONLY
|
||||
created_at: Optional[float] = None
|
||||
stats: ReplicationStats = field(default_factory=ReplicationStats)
|
||||
|
||||
sync_deletions: bool = True
|
||||
last_pull_at: Optional[float] = None
|
||||
filter_prefix: Optional[str] = None
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"bucket_name": self.bucket_name,
|
||||
@@ -140,8 +148,11 @@ class ReplicationRule:
|
||||
"mode": self.mode,
|
||||
"created_at": self.created_at,
|
||||
"stats": self.stats.to_dict(),
|
||||
"sync_deletions": self.sync_deletions,
|
||||
"last_pull_at": self.last_pull_at,
|
||||
"filter_prefix": self.filter_prefix,
|
||||
}
|
||||
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict) -> "ReplicationRule":
|
||||
stats_data = data.pop("stats", {})
|
||||
@@ -149,22 +160,28 @@ class ReplicationRule:
|
||||
data["mode"] = REPLICATION_MODE_NEW_ONLY
|
||||
if "created_at" not in data:
|
||||
data["created_at"] = None
|
||||
if "sync_deletions" not in data:
|
||||
data["sync_deletions"] = True
|
||||
if "last_pull_at" not in data:
|
||||
data["last_pull_at"] = None
|
||||
if "filter_prefix" not in data:
|
||||
data["filter_prefix"] = None
|
||||
rule = cls(**data)
|
||||
rule.stats = ReplicationStats.from_dict(stats_data) if stats_data else ReplicationStats()
|
||||
return rule
|
||||
|
||||
|
||||
class ReplicationFailureStore:
|
||||
MAX_FAILURES_PER_BUCKET = 50
|
||||
|
||||
def __init__(self, storage_root: Path) -> None:
|
||||
def __init__(self, storage_root: Path, max_failures_per_bucket: int = 50) -> None:
|
||||
self.storage_root = storage_root
|
||||
self.max_failures_per_bucket = max_failures_per_bucket
|
||||
self._lock = threading.Lock()
|
||||
self._cache: Dict[str, List[ReplicationFailure]] = {}
|
||||
|
||||
def _get_failures_path(self, bucket_name: str) -> Path:
|
||||
return self.storage_root / ".myfsio.sys" / "buckets" / bucket_name / "replication_failures.json"
|
||||
|
||||
def load_failures(self, bucket_name: str) -> List[ReplicationFailure]:
|
||||
def _load_from_disk(self, bucket_name: str) -> List[ReplicationFailure]:
|
||||
path = self._get_failures_path(bucket_name)
|
||||
if not path.exists():
|
||||
return []
|
||||
@@ -176,16 +193,28 @@ class ReplicationFailureStore:
|
||||
logger.error(f"Failed to load replication failures for {bucket_name}: {e}")
|
||||
return []
|
||||
|
||||
def save_failures(self, bucket_name: str, failures: List[ReplicationFailure]) -> None:
|
||||
def _save_to_disk(self, bucket_name: str, failures: List[ReplicationFailure]) -> None:
|
||||
path = self._get_failures_path(bucket_name)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
data = {"failures": [f.to_dict() for f in failures[:self.MAX_FAILURES_PER_BUCKET]]}
|
||||
data = {"failures": [f.to_dict() for f in failures[:self.max_failures_per_bucket]]}
|
||||
try:
|
||||
with open(path, "w") as f:
|
||||
json.dump(data, f, indent=2)
|
||||
except OSError as e:
|
||||
logger.error(f"Failed to save replication failures for {bucket_name}: {e}")
|
||||
|
||||
def load_failures(self, bucket_name: str) -> List[ReplicationFailure]:
|
||||
if bucket_name in self._cache:
|
||||
return list(self._cache[bucket_name])
|
||||
failures = self._load_from_disk(bucket_name)
|
||||
self._cache[bucket_name] = failures
|
||||
return list(failures)
|
||||
|
||||
def save_failures(self, bucket_name: str, failures: List[ReplicationFailure]) -> None:
|
||||
trimmed = failures[:self.max_failures_per_bucket]
|
||||
self._cache[bucket_name] = trimmed
|
||||
self._save_to_disk(bucket_name, trimmed)
|
||||
|
||||
def add_failure(self, bucket_name: str, failure: ReplicationFailure) -> None:
|
||||
with self._lock:
|
||||
failures = self.load_failures(bucket_name)
|
||||
@@ -211,6 +240,7 @@ class ReplicationFailureStore:
|
||||
|
||||
def clear_failures(self, bucket_name: str) -> None:
|
||||
with self._lock:
|
||||
self._cache.pop(bucket_name, None)
|
||||
path = self._get_failures_path(bucket_name)
|
||||
if path.exists():
|
||||
path.unlink()
|
||||
@@ -224,18 +254,43 @@ class ReplicationFailureStore:
|
||||
|
||||
|
||||
class ReplicationManager:
|
||||
def __init__(self, storage: ObjectStorage, connections: ConnectionStore, rules_path: Path, storage_root: Path) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
storage: ObjectStorage,
|
||||
connections: ConnectionStore,
|
||||
rules_path: Path,
|
||||
storage_root: Path,
|
||||
connect_timeout: int = 5,
|
||||
read_timeout: int = 30,
|
||||
max_retries: int = 2,
|
||||
streaming_threshold_bytes: int = 10 * 1024 * 1024,
|
||||
max_failures_per_bucket: int = 50,
|
||||
) -> None:
|
||||
self.storage = storage
|
||||
self.connections = connections
|
||||
self.rules_path = rules_path
|
||||
self.storage_root = storage_root
|
||||
self.connect_timeout = connect_timeout
|
||||
self.read_timeout = read_timeout
|
||||
self.max_retries = max_retries
|
||||
self.streaming_threshold_bytes = streaming_threshold_bytes
|
||||
self._rules: Dict[str, ReplicationRule] = {}
|
||||
self._stats_lock = threading.Lock()
|
||||
self._executor = ThreadPoolExecutor(max_workers=4, thread_name_prefix="ReplicationWorker")
|
||||
self._shutdown = False
|
||||
self.failure_store = ReplicationFailureStore(storage_root)
|
||||
self.failure_store = ReplicationFailureStore(storage_root, max_failures_per_bucket)
|
||||
self.reload_rules()
|
||||
|
||||
def _create_client(self, connection: RemoteConnection, *, health_check: bool = False) -> Any:
|
||||
"""Create an S3 client with the manager's configured timeouts."""
|
||||
return _create_s3_client(
|
||||
connection,
|
||||
health_check=health_check,
|
||||
connect_timeout=self.connect_timeout,
|
||||
read_timeout=self.read_timeout,
|
||||
max_retries=self.max_retries,
|
||||
)
|
||||
|
||||
def shutdown(self, wait: bool = True) -> None:
|
||||
"""Shutdown the replication executor gracefully.
|
||||
|
||||
@@ -271,7 +326,7 @@ class ReplicationManager:
|
||||
Uses short timeouts to prevent blocking.
|
||||
"""
|
||||
try:
|
||||
s3 = _create_s3_client(connection, health_check=True)
|
||||
s3 = self._create_client(connection, health_check=True)
|
||||
s3.list_buckets()
|
||||
return True
|
||||
except Exception as e:
|
||||
@@ -281,6 +336,9 @@ class ReplicationManager:
|
||||
def get_rule(self, bucket_name: str) -> Optional[ReplicationRule]:
|
||||
return self._rules.get(bucket_name)
|
||||
|
||||
def list_rules(self) -> List[ReplicationRule]:
|
||||
return list(self._rules.values())
|
||||
|
||||
def set_rule(self, rule: ReplicationRule) -> None:
|
||||
old_rule = self._rules.get(rule.bucket_name)
|
||||
was_all_mode = old_rule and old_rule.mode == REPLICATION_MODE_ALL if old_rule else False
|
||||
@@ -320,7 +378,7 @@ class ReplicationManager:
|
||||
source_objects = self.storage.list_objects_all(bucket_name)
|
||||
source_keys = {obj.key: obj.size for obj in source_objects}
|
||||
|
||||
s3 = _create_s3_client(connection)
|
||||
s3 = self._create_client(connection)
|
||||
|
||||
dest_keys = set()
|
||||
bytes_synced = 0
|
||||
@@ -386,7 +444,7 @@ class ReplicationManager:
|
||||
raise ValueError(f"Connection {connection_id} not found")
|
||||
|
||||
try:
|
||||
s3 = _create_s3_client(connection)
|
||||
s3 = self._create_client(connection)
|
||||
s3.create_bucket(Bucket=bucket_name)
|
||||
except ClientError as e:
|
||||
logger.error(f"Failed to create remote bucket {bucket_name}: {e}")
|
||||
@@ -429,7 +487,7 @@ class ReplicationManager:
|
||||
return
|
||||
|
||||
try:
|
||||
s3 = _create_s3_client(conn)
|
||||
s3 = self._create_client(conn)
|
||||
|
||||
if action == "delete":
|
||||
try:
|
||||
@@ -472,7 +530,7 @@ class ReplicationManager:
|
||||
if content_type:
|
||||
extra_args["ContentType"] = content_type
|
||||
|
||||
if file_size >= STREAMING_THRESHOLD_BYTES:
|
||||
if file_size >= self.streaming_threshold_bytes:
|
||||
s3.upload_file(
|
||||
str(path),
|
||||
rule.target_bucket,
|
||||
|
||||
1539
app/s3_api.py
1539
app/s3_api.py
File diff suppressed because it is too large
Load Diff
296
app/s3_client.py
Normal file
296
app/s3_client.py
Normal file
@@ -0,0 +1,296 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from typing import Any, Generator, Optional
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import ClientError, EndpointConnectionError, ConnectionClosedError
|
||||
from flask import current_app, session
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
UI_PROXY_USER_AGENT = "MyFSIO-UIProxy/1.0"
|
||||
|
||||
_BOTO_ERROR_MAP = {
|
||||
"NoSuchBucket": 404,
|
||||
"NoSuchKey": 404,
|
||||
"NoSuchUpload": 404,
|
||||
"BucketAlreadyExists": 409,
|
||||
"BucketAlreadyOwnedByYou": 409,
|
||||
"BucketNotEmpty": 409,
|
||||
"AccessDenied": 403,
|
||||
"InvalidAccessKeyId": 403,
|
||||
"SignatureDoesNotMatch": 403,
|
||||
"InvalidBucketName": 400,
|
||||
"InvalidArgument": 400,
|
||||
"MalformedXML": 400,
|
||||
"EntityTooLarge": 400,
|
||||
"QuotaExceeded": 403,
|
||||
}
|
||||
|
||||
_UPLOAD_REGISTRY_MAX_AGE = 86400
|
||||
_UPLOAD_REGISTRY_CLEANUP_INTERVAL = 3600
|
||||
|
||||
|
||||
class UploadRegistry:
|
||||
def __init__(self) -> None:
|
||||
self._entries: dict[str, tuple[str, str, float]] = {}
|
||||
self._lock = threading.Lock()
|
||||
self._last_cleanup = time.monotonic()
|
||||
|
||||
def register(self, upload_id: str, bucket_name: str, object_key: str) -> None:
|
||||
with self._lock:
|
||||
self._entries[upload_id] = (bucket_name, object_key, time.monotonic())
|
||||
self._maybe_cleanup()
|
||||
|
||||
def get_key(self, upload_id: str, bucket_name: str) -> Optional[str]:
|
||||
with self._lock:
|
||||
entry = self._entries.get(upload_id)
|
||||
if entry is None:
|
||||
return None
|
||||
stored_bucket, key, created_at = entry
|
||||
if stored_bucket != bucket_name:
|
||||
return None
|
||||
if time.monotonic() - created_at > _UPLOAD_REGISTRY_MAX_AGE:
|
||||
del self._entries[upload_id]
|
||||
return None
|
||||
return key
|
||||
|
||||
def remove(self, upload_id: str) -> None:
|
||||
with self._lock:
|
||||
self._entries.pop(upload_id, None)
|
||||
|
||||
def _maybe_cleanup(self) -> None:
|
||||
now = time.monotonic()
|
||||
if now - self._last_cleanup < _UPLOAD_REGISTRY_CLEANUP_INTERVAL:
|
||||
return
|
||||
self._last_cleanup = now
|
||||
cutoff = now - _UPLOAD_REGISTRY_MAX_AGE
|
||||
stale = [uid for uid, (_, _, ts) in self._entries.items() if ts < cutoff]
|
||||
for uid in stale:
|
||||
del self._entries[uid]
|
||||
|
||||
|
||||
class S3ProxyClient:
|
||||
def __init__(self, api_base_url: str, region: str = "us-east-1") -> None:
|
||||
if not api_base_url:
|
||||
raise ValueError("api_base_url is required for S3ProxyClient")
|
||||
self._api_base_url = api_base_url.rstrip("/")
|
||||
self._region = region
|
||||
self.upload_registry = UploadRegistry()
|
||||
|
||||
@property
|
||||
def api_base_url(self) -> str:
|
||||
return self._api_base_url
|
||||
|
||||
def get_client(self, access_key: str, secret_key: str) -> Any:
|
||||
if not access_key or not secret_key:
|
||||
raise ValueError("Both access_key and secret_key are required")
|
||||
config = Config(
|
||||
user_agent_extra=UI_PROXY_USER_AGENT,
|
||||
connect_timeout=5,
|
||||
read_timeout=30,
|
||||
retries={"max_attempts": 0},
|
||||
signature_version="s3v4",
|
||||
s3={"addressing_style": "path"},
|
||||
request_checksum_calculation="when_required",
|
||||
response_checksum_validation="when_required",
|
||||
)
|
||||
return boto3.client(
|
||||
"s3",
|
||||
endpoint_url=self._api_base_url,
|
||||
aws_access_key_id=access_key,
|
||||
aws_secret_access_key=secret_key,
|
||||
region_name=self._region,
|
||||
config=config,
|
||||
)
|
||||
|
||||
|
||||
def _get_proxy() -> S3ProxyClient:
|
||||
proxy = current_app.extensions.get("s3_proxy")
|
||||
if proxy is None:
|
||||
raise RuntimeError(
|
||||
"S3 proxy not configured. Set API_BASE_URL or run both API and UI servers."
|
||||
)
|
||||
return proxy
|
||||
|
||||
|
||||
def _get_session_creds() -> tuple[str, str]:
|
||||
secret_store = current_app.extensions["secret_store"]
|
||||
secret_store.purge_expired()
|
||||
token = session.get("cred_token")
|
||||
if not token:
|
||||
raise PermissionError("Not authenticated")
|
||||
creds = secret_store.peek(token)
|
||||
if not creds:
|
||||
raise PermissionError("Session expired")
|
||||
access_key = creds.get("access_key", "")
|
||||
secret_key = creds.get("secret_key", "")
|
||||
if not access_key or not secret_key:
|
||||
raise PermissionError("Invalid session credentials")
|
||||
return access_key, secret_key
|
||||
|
||||
|
||||
def get_session_s3_client() -> Any:
|
||||
proxy = _get_proxy()
|
||||
access_key, secret_key = _get_session_creds()
|
||||
return proxy.get_client(access_key, secret_key)
|
||||
|
||||
|
||||
def get_upload_registry() -> UploadRegistry:
|
||||
return _get_proxy().upload_registry
|
||||
|
||||
|
||||
def handle_client_error(exc: ClientError) -> tuple[dict[str, str], int]:
|
||||
error_info = exc.response.get("Error", {})
|
||||
code = error_info.get("Code", "InternalError")
|
||||
message = error_info.get("Message") or "S3 operation failed"
|
||||
http_status = _BOTO_ERROR_MAP.get(code)
|
||||
if http_status is None:
|
||||
http_status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode", 500)
|
||||
return {"error": message}, http_status
|
||||
|
||||
|
||||
def handle_connection_error(exc: Exception) -> tuple[dict[str, str], int]:
|
||||
logger.error("S3 API connection failed: %s", exc)
|
||||
return {"error": "S3 API server is unreachable. Ensure the API server is running."}, 502
|
||||
|
||||
|
||||
def format_datetime_display(dt: Any, display_tz: str = "UTC") -> str:
|
||||
from .ui import _format_datetime_display
|
||||
return _format_datetime_display(dt, display_tz)
|
||||
|
||||
|
||||
def format_datetime_iso(dt: Any, display_tz: str = "UTC") -> str:
|
||||
from .ui import _format_datetime_iso
|
||||
return _format_datetime_iso(dt, display_tz)
|
||||
|
||||
|
||||
def build_url_templates(bucket_name: str) -> dict[str, str]:
|
||||
from flask import url_for
|
||||
preview_t = url_for("ui.object_preview", bucket_name=bucket_name, object_key="KEY_PLACEHOLDER")
|
||||
delete_t = url_for("ui.delete_object", bucket_name=bucket_name, object_key="KEY_PLACEHOLDER")
|
||||
presign_t = url_for("ui.object_presign", bucket_name=bucket_name, object_key="KEY_PLACEHOLDER")
|
||||
versions_t = url_for("ui.object_versions", bucket_name=bucket_name, object_key="KEY_PLACEHOLDER")
|
||||
restore_t = url_for(
|
||||
"ui.restore_object_version",
|
||||
bucket_name=bucket_name,
|
||||
object_key="KEY_PLACEHOLDER",
|
||||
version_id="VERSION_ID_PLACEHOLDER",
|
||||
)
|
||||
tags_t = url_for("ui.object_tags", bucket_name=bucket_name, object_key="KEY_PLACEHOLDER")
|
||||
copy_t = url_for("ui.copy_object", bucket_name=bucket_name, object_key="KEY_PLACEHOLDER")
|
||||
move_t = url_for("ui.move_object", bucket_name=bucket_name, object_key="KEY_PLACEHOLDER")
|
||||
metadata_t = url_for("ui.object_metadata", bucket_name=bucket_name, object_key="KEY_PLACEHOLDER")
|
||||
return {
|
||||
"preview": preview_t,
|
||||
"download": preview_t + "?download=1",
|
||||
"presign": presign_t,
|
||||
"delete": delete_t,
|
||||
"versions": versions_t,
|
||||
"restore": restore_t,
|
||||
"tags": tags_t,
|
||||
"copy": copy_t,
|
||||
"move": move_t,
|
||||
"metadata": metadata_t,
|
||||
}
|
||||
|
||||
|
||||
def translate_list_objects(
|
||||
boto3_response: dict[str, Any],
|
||||
url_templates: dict[str, str],
|
||||
display_tz: str = "UTC",
|
||||
versioning_enabled: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
objects_data = []
|
||||
for obj in boto3_response.get("Contents", []):
|
||||
last_mod = obj["LastModified"]
|
||||
objects_data.append({
|
||||
"key": obj["Key"],
|
||||
"size": obj["Size"],
|
||||
"last_modified": last_mod.isoformat(),
|
||||
"last_modified_display": format_datetime_display(last_mod, display_tz),
|
||||
"last_modified_iso": format_datetime_iso(last_mod, display_tz),
|
||||
"etag": obj.get("ETag", "").strip('"'),
|
||||
})
|
||||
return {
|
||||
"objects": objects_data,
|
||||
"is_truncated": boto3_response.get("IsTruncated", False),
|
||||
"next_continuation_token": boto3_response.get("NextContinuationToken"),
|
||||
"total_count": boto3_response.get("KeyCount", len(objects_data)),
|
||||
"versioning_enabled": versioning_enabled,
|
||||
"url_templates": url_templates,
|
||||
}
|
||||
|
||||
|
||||
def get_versioning_via_s3(client: Any, bucket_name: str) -> bool:
|
||||
try:
|
||||
resp = client.get_bucket_versioning(Bucket=bucket_name)
|
||||
return resp.get("Status") == "Enabled"
|
||||
except ClientError as exc:
|
||||
code = exc.response.get("Error", {}).get("Code", "")
|
||||
if code != "NoSuchBucket":
|
||||
logger.warning("Failed to check versioning for %s: %s", bucket_name, code)
|
||||
return False
|
||||
|
||||
|
||||
def stream_objects_ndjson(
|
||||
client: Any,
|
||||
bucket_name: str,
|
||||
prefix: Optional[str],
|
||||
url_templates: dict[str, str],
|
||||
display_tz: str = "UTC",
|
||||
versioning_enabled: bool = False,
|
||||
delimiter: Optional[str] = None,
|
||||
) -> Generator[str, None, None]:
|
||||
meta_line = json.dumps({
|
||||
"type": "meta",
|
||||
"versioning_enabled": versioning_enabled,
|
||||
"url_templates": url_templates,
|
||||
}) + "\n"
|
||||
yield meta_line
|
||||
|
||||
yield json.dumps({"type": "count", "total_count": 0}) + "\n"
|
||||
|
||||
kwargs: dict[str, Any] = {"Bucket": bucket_name, "MaxKeys": 1000}
|
||||
if prefix:
|
||||
kwargs["Prefix"] = prefix
|
||||
if delimiter:
|
||||
kwargs["Delimiter"] = delimiter
|
||||
|
||||
running_count = 0
|
||||
try:
|
||||
paginator = client.get_paginator("list_objects_v2")
|
||||
for page in paginator.paginate(**kwargs):
|
||||
for cp in page.get("CommonPrefixes", []):
|
||||
yield json.dumps({
|
||||
"type": "folder",
|
||||
"prefix": cp["Prefix"],
|
||||
}) + "\n"
|
||||
page_contents = page.get("Contents", [])
|
||||
for obj in page_contents:
|
||||
last_mod = obj["LastModified"]
|
||||
yield json.dumps({
|
||||
"type": "object",
|
||||
"key": obj["Key"],
|
||||
"size": obj["Size"],
|
||||
"last_modified": last_mod.isoformat(),
|
||||
"last_modified_display": format_datetime_display(last_mod, display_tz),
|
||||
"last_modified_iso": format_datetime_iso(last_mod, display_tz),
|
||||
"etag": obj.get("ETag", "").strip('"'),
|
||||
}) + "\n"
|
||||
running_count += len(page_contents)
|
||||
yield json.dumps({"type": "count", "total_count": running_count}) + "\n"
|
||||
except ClientError as exc:
|
||||
error_msg = exc.response.get("Error", {}).get("Message", "S3 operation failed")
|
||||
yield json.dumps({"type": "error", "error": error_msg}) + "\n"
|
||||
return
|
||||
except (EndpointConnectionError, ConnectionClosedError):
|
||||
yield json.dumps({"type": "error", "error": "S3 API server is unreachable"}) + "\n"
|
||||
return
|
||||
|
||||
yield json.dumps({"type": "done"}) + "\n"
|
||||
@@ -18,6 +18,18 @@ class EphemeralSecretStore:
|
||||
self._store[token] = (payload, expires_at)
|
||||
return token
|
||||
|
||||
def peek(self, token: str | None) -> Any | None:
|
||||
if not token:
|
||||
return None
|
||||
entry = self._store.get(token)
|
||||
if not entry:
|
||||
return None
|
||||
payload, expires_at = entry
|
||||
if expires_at < time.time():
|
||||
self._store.pop(token, None)
|
||||
return None
|
||||
return payload
|
||||
|
||||
def pop(self, token: str | None) -> Any | None:
|
||||
if not token:
|
||||
return None
|
||||
|
||||
171
app/select_content.py
Normal file
171
app/select_content.py
Normal file
@@ -0,0 +1,171 @@
|
||||
"""S3 SelectObjectContent SQL query execution using DuckDB."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Generator, Optional
|
||||
|
||||
try:
|
||||
import duckdb
|
||||
DUCKDB_AVAILABLE = True
|
||||
except ImportError:
|
||||
DUCKDB_AVAILABLE = False
|
||||
|
||||
|
||||
class SelectError(Exception):
|
||||
"""Error during SELECT query execution."""
|
||||
pass
|
||||
|
||||
|
||||
def execute_select_query(
|
||||
file_path: Path,
|
||||
expression: str,
|
||||
input_format: str,
|
||||
input_config: Dict[str, Any],
|
||||
output_format: str,
|
||||
output_config: Dict[str, Any],
|
||||
chunk_size: int = 65536,
|
||||
) -> Generator[bytes, None, None]:
|
||||
"""Execute SQL query on object content."""
|
||||
if not DUCKDB_AVAILABLE:
|
||||
raise SelectError("DuckDB is not installed. Install with: pip install duckdb")
|
||||
|
||||
conn = duckdb.connect(":memory:")
|
||||
|
||||
try:
|
||||
if input_format == "CSV":
|
||||
_load_csv(conn, file_path, input_config)
|
||||
elif input_format == "JSON":
|
||||
_load_json(conn, file_path, input_config)
|
||||
elif input_format == "Parquet":
|
||||
_load_parquet(conn, file_path)
|
||||
else:
|
||||
raise SelectError(f"Unsupported input format: {input_format}")
|
||||
|
||||
normalized_expression = expression.replace("s3object", "data").replace("S3Object", "data")
|
||||
|
||||
try:
|
||||
result = conn.execute(normalized_expression)
|
||||
except duckdb.Error as exc:
|
||||
raise SelectError(f"SQL execution error: {exc}")
|
||||
|
||||
if output_format == "CSV":
|
||||
yield from _output_csv(result, output_config, chunk_size)
|
||||
elif output_format == "JSON":
|
||||
yield from _output_json(result, output_config, chunk_size)
|
||||
else:
|
||||
raise SelectError(f"Unsupported output format: {output_format}")
|
||||
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _load_csv(conn, file_path: Path, config: Dict[str, Any]) -> None:
|
||||
"""Load CSV file into DuckDB."""
|
||||
file_header_info = config.get("file_header_info", "NONE")
|
||||
delimiter = config.get("field_delimiter", ",")
|
||||
quote = config.get("quote_character", '"')
|
||||
|
||||
header = file_header_info in ("USE", "IGNORE")
|
||||
path_str = str(file_path).replace("\\", "/")
|
||||
|
||||
conn.execute(f"""
|
||||
CREATE TABLE data AS
|
||||
SELECT * FROM read_csv('{path_str}',
|
||||
header={header},
|
||||
delim='{delimiter}',
|
||||
quote='{quote}'
|
||||
)
|
||||
""")
|
||||
|
||||
|
||||
def _load_json(conn, file_path: Path, config: Dict[str, Any]) -> None:
|
||||
"""Load JSON file into DuckDB."""
|
||||
json_type = config.get("type", "DOCUMENT")
|
||||
path_str = str(file_path).replace("\\", "/")
|
||||
|
||||
if json_type == "LINES":
|
||||
conn.execute(f"""
|
||||
CREATE TABLE data AS
|
||||
SELECT * FROM read_json_auto('{path_str}', format='newline_delimited')
|
||||
""")
|
||||
else:
|
||||
conn.execute(f"""
|
||||
CREATE TABLE data AS
|
||||
SELECT * FROM read_json_auto('{path_str}', format='array')
|
||||
""")
|
||||
|
||||
|
||||
def _load_parquet(conn, file_path: Path) -> None:
|
||||
"""Load Parquet file into DuckDB."""
|
||||
path_str = str(file_path).replace("\\", "/")
|
||||
conn.execute(f"CREATE TABLE data AS SELECT * FROM read_parquet('{path_str}')")
|
||||
|
||||
|
||||
def _output_csv(
|
||||
result,
|
||||
config: Dict[str, Any],
|
||||
chunk_size: int,
|
||||
) -> Generator[bytes, None, None]:
|
||||
"""Output query results as CSV."""
|
||||
delimiter = config.get("field_delimiter", ",")
|
||||
record_delimiter = config.get("record_delimiter", "\n")
|
||||
quote = config.get("quote_character", '"')
|
||||
|
||||
buffer = ""
|
||||
|
||||
while True:
|
||||
rows = result.fetchmany(1000)
|
||||
if not rows:
|
||||
break
|
||||
|
||||
for row in rows:
|
||||
fields = []
|
||||
for value in row:
|
||||
if value is None:
|
||||
fields.append("")
|
||||
elif isinstance(value, str):
|
||||
if delimiter in value or quote in value or record_delimiter in value:
|
||||
escaped = value.replace(quote, quote + quote)
|
||||
fields.append(f'{quote}{escaped}{quote}')
|
||||
else:
|
||||
fields.append(value)
|
||||
else:
|
||||
fields.append(str(value))
|
||||
|
||||
buffer += delimiter.join(fields) + record_delimiter
|
||||
|
||||
while len(buffer) >= chunk_size:
|
||||
yield buffer[:chunk_size].encode("utf-8")
|
||||
buffer = buffer[chunk_size:]
|
||||
|
||||
if buffer:
|
||||
yield buffer.encode("utf-8")
|
||||
|
||||
|
||||
def _output_json(
|
||||
result,
|
||||
config: Dict[str, Any],
|
||||
chunk_size: int,
|
||||
) -> Generator[bytes, None, None]:
|
||||
"""Output query results as JSON Lines."""
|
||||
record_delimiter = config.get("record_delimiter", "\n")
|
||||
columns = [desc[0] for desc in result.description]
|
||||
|
||||
buffer = ""
|
||||
|
||||
while True:
|
||||
rows = result.fetchmany(1000)
|
||||
if not rows:
|
||||
break
|
||||
|
||||
for row in rows:
|
||||
record = dict(zip(columns, row))
|
||||
buffer += json.dumps(record, default=str) + record_delimiter
|
||||
|
||||
while len(buffer) >= chunk_size:
|
||||
yield buffer[:chunk_size].encode("utf-8")
|
||||
buffer = buffer[chunk_size:]
|
||||
|
||||
if buffer:
|
||||
yield buffer.encode("utf-8")
|
||||
177
app/site_registry.py
Normal file
177
app/site_registry.py
Normal file
@@ -0,0 +1,177 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
|
||||
@dataclass
|
||||
class SiteInfo:
|
||||
site_id: str
|
||||
endpoint: str
|
||||
region: str = "us-east-1"
|
||||
priority: int = 100
|
||||
display_name: str = ""
|
||||
created_at: Optional[float] = None
|
||||
updated_at: Optional[float] = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not self.display_name:
|
||||
self.display_name = self.site_id
|
||||
if self.created_at is None:
|
||||
self.created_at = time.time()
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"site_id": self.site_id,
|
||||
"endpoint": self.endpoint,
|
||||
"region": self.region,
|
||||
"priority": self.priority,
|
||||
"display_name": self.display_name,
|
||||
"created_at": self.created_at,
|
||||
"updated_at": self.updated_at,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> SiteInfo:
|
||||
return cls(
|
||||
site_id=data["site_id"],
|
||||
endpoint=data.get("endpoint", ""),
|
||||
region=data.get("region", "us-east-1"),
|
||||
priority=data.get("priority", 100),
|
||||
display_name=data.get("display_name", ""),
|
||||
created_at=data.get("created_at"),
|
||||
updated_at=data.get("updated_at"),
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class PeerSite:
|
||||
site_id: str
|
||||
endpoint: str
|
||||
region: str = "us-east-1"
|
||||
priority: int = 100
|
||||
display_name: str = ""
|
||||
created_at: Optional[float] = None
|
||||
updated_at: Optional[float] = None
|
||||
connection_id: Optional[str] = None
|
||||
is_healthy: Optional[bool] = None
|
||||
last_health_check: Optional[float] = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not self.display_name:
|
||||
self.display_name = self.site_id
|
||||
if self.created_at is None:
|
||||
self.created_at = time.time()
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"site_id": self.site_id,
|
||||
"endpoint": self.endpoint,
|
||||
"region": self.region,
|
||||
"priority": self.priority,
|
||||
"display_name": self.display_name,
|
||||
"created_at": self.created_at,
|
||||
"updated_at": self.updated_at,
|
||||
"connection_id": self.connection_id,
|
||||
"is_healthy": self.is_healthy,
|
||||
"last_health_check": self.last_health_check,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> PeerSite:
|
||||
return cls(
|
||||
site_id=data["site_id"],
|
||||
endpoint=data.get("endpoint", ""),
|
||||
region=data.get("region", "us-east-1"),
|
||||
priority=data.get("priority", 100),
|
||||
display_name=data.get("display_name", ""),
|
||||
created_at=data.get("created_at"),
|
||||
updated_at=data.get("updated_at"),
|
||||
connection_id=data.get("connection_id"),
|
||||
is_healthy=data.get("is_healthy"),
|
||||
last_health_check=data.get("last_health_check"),
|
||||
)
|
||||
|
||||
|
||||
class SiteRegistry:
|
||||
def __init__(self, config_path: Path) -> None:
|
||||
self.config_path = config_path
|
||||
self._local_site: Optional[SiteInfo] = None
|
||||
self._peers: Dict[str, PeerSite] = {}
|
||||
self.reload()
|
||||
|
||||
def reload(self) -> None:
|
||||
if not self.config_path.exists():
|
||||
self._local_site = None
|
||||
self._peers = {}
|
||||
return
|
||||
|
||||
try:
|
||||
with open(self.config_path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
|
||||
if data.get("local"):
|
||||
self._local_site = SiteInfo.from_dict(data["local"])
|
||||
else:
|
||||
self._local_site = None
|
||||
|
||||
self._peers = {}
|
||||
for peer_data in data.get("peers", []):
|
||||
peer = PeerSite.from_dict(peer_data)
|
||||
self._peers[peer.site_id] = peer
|
||||
|
||||
except (OSError, json.JSONDecodeError, KeyError):
|
||||
self._local_site = None
|
||||
self._peers = {}
|
||||
|
||||
def save(self) -> None:
|
||||
self.config_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
data = {
|
||||
"local": self._local_site.to_dict() if self._local_site else None,
|
||||
"peers": [peer.to_dict() for peer in self._peers.values()],
|
||||
}
|
||||
with open(self.config_path, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, indent=2)
|
||||
|
||||
def get_local_site(self) -> Optional[SiteInfo]:
|
||||
return self._local_site
|
||||
|
||||
def set_local_site(self, site: SiteInfo) -> None:
|
||||
site.updated_at = time.time()
|
||||
self._local_site = site
|
||||
self.save()
|
||||
|
||||
def list_peers(self) -> List[PeerSite]:
|
||||
return list(self._peers.values())
|
||||
|
||||
def get_peer(self, site_id: str) -> Optional[PeerSite]:
|
||||
return self._peers.get(site_id)
|
||||
|
||||
def add_peer(self, peer: PeerSite) -> None:
|
||||
peer.created_at = peer.created_at or time.time()
|
||||
self._peers[peer.site_id] = peer
|
||||
self.save()
|
||||
|
||||
def update_peer(self, peer: PeerSite) -> None:
|
||||
if peer.site_id not in self._peers:
|
||||
raise ValueError(f"Peer {peer.site_id} not found")
|
||||
peer.updated_at = time.time()
|
||||
self._peers[peer.site_id] = peer
|
||||
self.save()
|
||||
|
||||
def delete_peer(self, site_id: str) -> bool:
|
||||
if site_id in self._peers:
|
||||
del self._peers[site_id]
|
||||
self.save()
|
||||
return True
|
||||
return False
|
||||
|
||||
def update_health(self, site_id: str, is_healthy: bool) -> None:
|
||||
peer = self._peers.get(site_id)
|
||||
if peer:
|
||||
peer.is_healthy = is_healthy
|
||||
peer.last_health_check = time.time()
|
||||
self.save()
|
||||
416
app/site_sync.py
Normal file
416
app/site_sync.py
Normal file
@@ -0,0 +1,416 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, TYPE_CHECKING
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .connections import ConnectionStore, RemoteConnection
|
||||
from .replication import ReplicationManager, ReplicationRule
|
||||
from .storage import ObjectStorage
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SITE_SYNC_USER_AGENT = "SiteSyncAgent/1.0"
|
||||
|
||||
|
||||
@dataclass
|
||||
class SyncedObjectInfo:
|
||||
last_synced_at: float
|
||||
remote_etag: str
|
||||
source: str
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"last_synced_at": self.last_synced_at,
|
||||
"remote_etag": self.remote_etag,
|
||||
"source": self.source,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "SyncedObjectInfo":
|
||||
return cls(
|
||||
last_synced_at=data["last_synced_at"],
|
||||
remote_etag=data["remote_etag"],
|
||||
source=data["source"],
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SyncState:
|
||||
synced_objects: Dict[str, SyncedObjectInfo] = field(default_factory=dict)
|
||||
last_full_sync: Optional[float] = None
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"synced_objects": {k: v.to_dict() for k, v in self.synced_objects.items()},
|
||||
"last_full_sync": self.last_full_sync,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "SyncState":
|
||||
synced_objects = {}
|
||||
for k, v in data.get("synced_objects", {}).items():
|
||||
synced_objects[k] = SyncedObjectInfo.from_dict(v)
|
||||
return cls(
|
||||
synced_objects=synced_objects,
|
||||
last_full_sync=data.get("last_full_sync"),
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SiteSyncStats:
|
||||
last_sync_at: Optional[float] = None
|
||||
objects_pulled: int = 0
|
||||
objects_skipped: int = 0
|
||||
conflicts_resolved: int = 0
|
||||
deletions_applied: int = 0
|
||||
errors: int = 0
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"last_sync_at": self.last_sync_at,
|
||||
"objects_pulled": self.objects_pulled,
|
||||
"objects_skipped": self.objects_skipped,
|
||||
"conflicts_resolved": self.conflicts_resolved,
|
||||
"deletions_applied": self.deletions_applied,
|
||||
"errors": self.errors,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class RemoteObjectMeta:
|
||||
key: str
|
||||
size: int
|
||||
last_modified: datetime
|
||||
etag: str
|
||||
|
||||
@classmethod
|
||||
def from_s3_object(cls, obj: Dict[str, Any]) -> "RemoteObjectMeta":
|
||||
return cls(
|
||||
key=obj["Key"],
|
||||
size=obj.get("Size", 0),
|
||||
last_modified=obj["LastModified"],
|
||||
etag=obj.get("ETag", "").strip('"'),
|
||||
)
|
||||
|
||||
|
||||
def _create_sync_client(
|
||||
connection: "RemoteConnection",
|
||||
*,
|
||||
connect_timeout: int = 10,
|
||||
read_timeout: int = 120,
|
||||
max_retries: int = 2,
|
||||
) -> Any:
|
||||
config = Config(
|
||||
user_agent_extra=SITE_SYNC_USER_AGENT,
|
||||
connect_timeout=connect_timeout,
|
||||
read_timeout=read_timeout,
|
||||
retries={"max_attempts": max_retries},
|
||||
signature_version="s3v4",
|
||||
s3={"addressing_style": "path"},
|
||||
request_checksum_calculation="when_required",
|
||||
response_checksum_validation="when_required",
|
||||
)
|
||||
return boto3.client(
|
||||
"s3",
|
||||
endpoint_url=connection.endpoint_url,
|
||||
aws_access_key_id=connection.access_key,
|
||||
aws_secret_access_key=connection.secret_key,
|
||||
region_name=connection.region or "us-east-1",
|
||||
config=config,
|
||||
)
|
||||
|
||||
|
||||
class SiteSyncWorker:
|
||||
def __init__(
|
||||
self,
|
||||
storage: "ObjectStorage",
|
||||
connections: "ConnectionStore",
|
||||
replication_manager: "ReplicationManager",
|
||||
storage_root: Path,
|
||||
interval_seconds: int = 60,
|
||||
batch_size: int = 100,
|
||||
connect_timeout: int = 10,
|
||||
read_timeout: int = 120,
|
||||
max_retries: int = 2,
|
||||
clock_skew_tolerance_seconds: float = 1.0,
|
||||
):
|
||||
self.storage = storage
|
||||
self.connections = connections
|
||||
self.replication_manager = replication_manager
|
||||
self.storage_root = storage_root
|
||||
self.interval_seconds = interval_seconds
|
||||
self.batch_size = batch_size
|
||||
self.connect_timeout = connect_timeout
|
||||
self.read_timeout = read_timeout
|
||||
self.max_retries = max_retries
|
||||
self.clock_skew_tolerance_seconds = clock_skew_tolerance_seconds
|
||||
self._lock = threading.Lock()
|
||||
self._shutdown = threading.Event()
|
||||
self._sync_thread: Optional[threading.Thread] = None
|
||||
self._bucket_stats: Dict[str, SiteSyncStats] = {}
|
||||
|
||||
def _create_client(self, connection: "RemoteConnection") -> Any:
|
||||
"""Create an S3 client with the worker's configured timeouts."""
|
||||
return _create_sync_client(
|
||||
connection,
|
||||
connect_timeout=self.connect_timeout,
|
||||
read_timeout=self.read_timeout,
|
||||
max_retries=self.max_retries,
|
||||
)
|
||||
|
||||
def start(self) -> None:
|
||||
if self._sync_thread is not None and self._sync_thread.is_alive():
|
||||
return
|
||||
self._shutdown.clear()
|
||||
self._sync_thread = threading.Thread(
|
||||
target=self._sync_loop, name="site-sync-worker", daemon=True
|
||||
)
|
||||
self._sync_thread.start()
|
||||
logger.info("Site sync worker started (interval=%ds)", self.interval_seconds)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self._shutdown.set()
|
||||
if self._sync_thread is not None:
|
||||
self._sync_thread.join(timeout=10.0)
|
||||
logger.info("Site sync worker shut down")
|
||||
|
||||
def trigger_sync(self, bucket_name: str) -> Optional[SiteSyncStats]:
|
||||
from .replication import REPLICATION_MODE_BIDIRECTIONAL
|
||||
rule = self.replication_manager.get_rule(bucket_name)
|
||||
if not rule or rule.mode != REPLICATION_MODE_BIDIRECTIONAL or not rule.enabled:
|
||||
return None
|
||||
return self._sync_bucket(rule)
|
||||
|
||||
def get_stats(self, bucket_name: str) -> Optional[SiteSyncStats]:
|
||||
with self._lock:
|
||||
return self._bucket_stats.get(bucket_name)
|
||||
|
||||
def _sync_loop(self) -> None:
|
||||
while not self._shutdown.is_set():
|
||||
self._shutdown.wait(timeout=self.interval_seconds)
|
||||
if self._shutdown.is_set():
|
||||
break
|
||||
self._run_sync_cycle()
|
||||
|
||||
def _run_sync_cycle(self) -> None:
|
||||
from .replication import REPLICATION_MODE_BIDIRECTIONAL
|
||||
for bucket_name, rule in list(self.replication_manager._rules.items()):
|
||||
if self._shutdown.is_set():
|
||||
break
|
||||
if rule.mode != REPLICATION_MODE_BIDIRECTIONAL or not rule.enabled:
|
||||
continue
|
||||
try:
|
||||
stats = self._sync_bucket(rule)
|
||||
with self._lock:
|
||||
self._bucket_stats[bucket_name] = stats
|
||||
except Exception as e:
|
||||
logger.exception("Site sync failed for bucket %s: %s", bucket_name, e)
|
||||
|
||||
def _sync_bucket(self, rule: "ReplicationRule") -> SiteSyncStats:
|
||||
stats = SiteSyncStats()
|
||||
connection = self.connections.get(rule.target_connection_id)
|
||||
if not connection:
|
||||
logger.warning("Connection %s not found for bucket %s", rule.target_connection_id, rule.bucket_name)
|
||||
stats.errors += 1
|
||||
return stats
|
||||
|
||||
try:
|
||||
local_objects = self._list_local_objects(rule.bucket_name)
|
||||
except Exception as e:
|
||||
logger.error("Failed to list local objects for %s: %s", rule.bucket_name, e)
|
||||
stats.errors += 1
|
||||
return stats
|
||||
|
||||
try:
|
||||
remote_objects = self._list_remote_objects(rule, connection)
|
||||
except Exception as e:
|
||||
logger.error("Failed to list remote objects for %s: %s", rule.bucket_name, e)
|
||||
stats.errors += 1
|
||||
return stats
|
||||
|
||||
sync_state = self._load_sync_state(rule.bucket_name)
|
||||
local_keys = set(local_objects.keys())
|
||||
remote_keys = set(remote_objects.keys())
|
||||
|
||||
to_pull = []
|
||||
for key in remote_keys:
|
||||
remote_meta = remote_objects[key]
|
||||
local_meta = local_objects.get(key)
|
||||
if local_meta is None:
|
||||
to_pull.append(key)
|
||||
else:
|
||||
resolution = self._resolve_conflict(local_meta, remote_meta)
|
||||
if resolution == "pull":
|
||||
to_pull.append(key)
|
||||
stats.conflicts_resolved += 1
|
||||
else:
|
||||
stats.objects_skipped += 1
|
||||
|
||||
pulled_count = 0
|
||||
for key in to_pull:
|
||||
if self._shutdown.is_set():
|
||||
break
|
||||
if pulled_count >= self.batch_size:
|
||||
break
|
||||
remote_meta = remote_objects[key]
|
||||
success = self._pull_object(rule, key, connection, remote_meta)
|
||||
if success:
|
||||
stats.objects_pulled += 1
|
||||
pulled_count += 1
|
||||
sync_state.synced_objects[key] = SyncedObjectInfo(
|
||||
last_synced_at=time.time(),
|
||||
remote_etag=remote_meta.etag,
|
||||
source="remote",
|
||||
)
|
||||
else:
|
||||
stats.errors += 1
|
||||
|
||||
if rule.sync_deletions:
|
||||
for key in list(sync_state.synced_objects.keys()):
|
||||
if key not in remote_keys and key in local_keys:
|
||||
tracked = sync_state.synced_objects[key]
|
||||
if tracked.source == "remote":
|
||||
local_meta = local_objects.get(key)
|
||||
if local_meta and local_meta.last_modified.timestamp() <= tracked.last_synced_at:
|
||||
success = self._apply_remote_deletion(rule.bucket_name, key)
|
||||
if success:
|
||||
stats.deletions_applied += 1
|
||||
del sync_state.synced_objects[key]
|
||||
|
||||
sync_state.last_full_sync = time.time()
|
||||
self._save_sync_state(rule.bucket_name, sync_state)
|
||||
|
||||
with self.replication_manager._stats_lock:
|
||||
rule.last_pull_at = time.time()
|
||||
self.replication_manager.save_rules()
|
||||
|
||||
stats.last_sync_at = time.time()
|
||||
logger.info(
|
||||
"Site sync completed for %s: pulled=%d, skipped=%d, conflicts=%d, deletions=%d, errors=%d",
|
||||
rule.bucket_name,
|
||||
stats.objects_pulled,
|
||||
stats.objects_skipped,
|
||||
stats.conflicts_resolved,
|
||||
stats.deletions_applied,
|
||||
stats.errors,
|
||||
)
|
||||
return stats
|
||||
|
||||
def _list_local_objects(self, bucket_name: str) -> Dict[str, Any]:
|
||||
from .storage import ObjectMeta
|
||||
objects = self.storage.list_objects_all(bucket_name)
|
||||
return {obj.key: obj for obj in objects}
|
||||
|
||||
def _list_remote_objects(self, rule: "ReplicationRule", connection: "RemoteConnection") -> Dict[str, RemoteObjectMeta]:
|
||||
s3 = self._create_client(connection)
|
||||
result: Dict[str, RemoteObjectMeta] = {}
|
||||
paginator = s3.get_paginator("list_objects_v2")
|
||||
try:
|
||||
for page in paginator.paginate(Bucket=rule.target_bucket):
|
||||
for obj in page.get("Contents", []):
|
||||
meta = RemoteObjectMeta.from_s3_object(obj)
|
||||
result[meta.key] = meta
|
||||
except ClientError as e:
|
||||
if e.response["Error"]["Code"] == "NoSuchBucket":
|
||||
return {}
|
||||
raise
|
||||
return result
|
||||
|
||||
def _resolve_conflict(self, local_meta: Any, remote_meta: RemoteObjectMeta) -> str:
|
||||
local_ts = local_meta.last_modified.timestamp()
|
||||
remote_ts = remote_meta.last_modified.timestamp()
|
||||
|
||||
if abs(remote_ts - local_ts) < self.clock_skew_tolerance_seconds:
|
||||
local_etag = local_meta.etag or ""
|
||||
if remote_meta.etag == local_etag:
|
||||
return "skip"
|
||||
return "pull" if remote_meta.etag > local_etag else "keep"
|
||||
|
||||
return "pull" if remote_ts > local_ts else "keep"
|
||||
|
||||
def _pull_object(
|
||||
self,
|
||||
rule: "ReplicationRule",
|
||||
object_key: str,
|
||||
connection: "RemoteConnection",
|
||||
remote_meta: RemoteObjectMeta,
|
||||
) -> bool:
|
||||
s3 = self._create_client(connection)
|
||||
tmp_path = None
|
||||
try:
|
||||
tmp_dir = self.storage_root / ".myfsio.sys" / "tmp"
|
||||
tmp_dir.mkdir(parents=True, exist_ok=True)
|
||||
with tempfile.NamedTemporaryFile(dir=tmp_dir, delete=False) as tmp_file:
|
||||
tmp_path = Path(tmp_file.name)
|
||||
|
||||
s3.download_file(rule.target_bucket, object_key, str(tmp_path))
|
||||
|
||||
head_response = s3.head_object(Bucket=rule.target_bucket, Key=object_key)
|
||||
user_metadata = head_response.get("Metadata", {})
|
||||
|
||||
with open(tmp_path, "rb") as f:
|
||||
self.storage.put_object(
|
||||
rule.bucket_name,
|
||||
object_key,
|
||||
f,
|
||||
metadata=user_metadata if user_metadata else None,
|
||||
)
|
||||
|
||||
logger.debug("Pulled object %s/%s from remote", rule.bucket_name, object_key)
|
||||
return True
|
||||
|
||||
except ClientError as e:
|
||||
logger.error("Failed to pull %s/%s: %s", rule.bucket_name, object_key, e)
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error("Failed to store pulled object %s/%s: %s", rule.bucket_name, object_key, e)
|
||||
return False
|
||||
finally:
|
||||
if tmp_path and tmp_path.exists():
|
||||
try:
|
||||
tmp_path.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def _apply_remote_deletion(self, bucket_name: str, object_key: str) -> bool:
|
||||
try:
|
||||
self.storage.delete_object(bucket_name, object_key)
|
||||
logger.debug("Applied remote deletion for %s/%s", bucket_name, object_key)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error("Failed to apply remote deletion for %s/%s: %s", bucket_name, object_key, e)
|
||||
return False
|
||||
|
||||
def _sync_state_path(self, bucket_name: str) -> Path:
|
||||
return self.storage_root / ".myfsio.sys" / "buckets" / bucket_name / "site_sync_state.json"
|
||||
|
||||
def _load_sync_state(self, bucket_name: str) -> SyncState:
|
||||
path = self._sync_state_path(bucket_name)
|
||||
if not path.exists():
|
||||
return SyncState()
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
return SyncState.from_dict(data)
|
||||
except (json.JSONDecodeError, OSError, KeyError) as e:
|
||||
logger.warning("Failed to load sync state for %s: %s", bucket_name, e)
|
||||
return SyncState()
|
||||
|
||||
def _save_sync_state(self, bucket_name: str, state: SyncState) -> None:
|
||||
path = self._sync_state_path(bucket_name)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
path.write_text(json.dumps(state.to_dict(), indent=2), encoding="utf-8")
|
||||
except OSError as e:
|
||||
logger.warning("Failed to save sync state for %s: %s", bucket_name, e)
|
||||
1191
app/storage.py
1191
app/storage.py
File diff suppressed because it is too large
Load Diff
215
app/system_metrics.py
Normal file
215
app/system_metrics.py
Normal file
@@ -0,0 +1,215 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, TYPE_CHECKING
|
||||
|
||||
import psutil
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .storage import ObjectStorage
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SystemMetricsSnapshot:
|
||||
timestamp: datetime
|
||||
cpu_percent: float
|
||||
memory_percent: float
|
||||
disk_percent: float
|
||||
storage_bytes: int
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"timestamp": self.timestamp.strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||
"cpu_percent": round(self.cpu_percent, 2),
|
||||
"memory_percent": round(self.memory_percent, 2),
|
||||
"disk_percent": round(self.disk_percent, 2),
|
||||
"storage_bytes": self.storage_bytes,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "SystemMetricsSnapshot":
|
||||
timestamp_str = data["timestamp"]
|
||||
if timestamp_str.endswith("Z"):
|
||||
timestamp_str = timestamp_str[:-1] + "+00:00"
|
||||
return cls(
|
||||
timestamp=datetime.fromisoformat(timestamp_str),
|
||||
cpu_percent=data.get("cpu_percent", 0.0),
|
||||
memory_percent=data.get("memory_percent", 0.0),
|
||||
disk_percent=data.get("disk_percent", 0.0),
|
||||
storage_bytes=data.get("storage_bytes", 0),
|
||||
)
|
||||
|
||||
|
||||
class SystemMetricsCollector:
|
||||
def __init__(
|
||||
self,
|
||||
storage_root: Path,
|
||||
interval_minutes: int = 5,
|
||||
retention_hours: int = 24,
|
||||
):
|
||||
self.storage_root = storage_root
|
||||
self.interval_seconds = interval_minutes * 60
|
||||
self.retention_hours = retention_hours
|
||||
self._lock = threading.Lock()
|
||||
self._shutdown = threading.Event()
|
||||
self._snapshots: List[SystemMetricsSnapshot] = []
|
||||
self._storage_ref: Optional["ObjectStorage"] = None
|
||||
|
||||
self._load_history()
|
||||
|
||||
self._snapshot_thread = threading.Thread(
|
||||
target=self._snapshot_loop,
|
||||
name="system-metrics-snapshot",
|
||||
daemon=True,
|
||||
)
|
||||
self._snapshot_thread.start()
|
||||
|
||||
def set_storage(self, storage: "ObjectStorage") -> None:
|
||||
with self._lock:
|
||||
self._storage_ref = storage
|
||||
|
||||
def _config_path(self) -> Path:
|
||||
return self.storage_root / ".myfsio.sys" / "config" / "metrics_history.json"
|
||||
|
||||
def _load_history(self) -> None:
|
||||
config_path = self._config_path()
|
||||
if not config_path.exists():
|
||||
return
|
||||
try:
|
||||
data = json.loads(config_path.read_text(encoding="utf-8"))
|
||||
history_data = data.get("history", [])
|
||||
self._snapshots = [SystemMetricsSnapshot.from_dict(s) for s in history_data]
|
||||
self._prune_old_snapshots()
|
||||
except (json.JSONDecodeError, OSError, KeyError) as e:
|
||||
logger.warning(f"Failed to load system metrics history: {e}")
|
||||
|
||||
def _save_history(self) -> None:
|
||||
config_path = self._config_path()
|
||||
config_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
data = {"history": [s.to_dict() for s in self._snapshots]}
|
||||
config_path.write_text(json.dumps(data, indent=2), encoding="utf-8")
|
||||
except OSError as e:
|
||||
logger.warning(f"Failed to save system metrics history: {e}")
|
||||
|
||||
def _prune_old_snapshots(self) -> None:
|
||||
if not self._snapshots:
|
||||
return
|
||||
cutoff = datetime.now(timezone.utc).timestamp() - (self.retention_hours * 3600)
|
||||
self._snapshots = [
|
||||
s for s in self._snapshots if s.timestamp.timestamp() > cutoff
|
||||
]
|
||||
|
||||
def _snapshot_loop(self) -> None:
|
||||
while not self._shutdown.is_set():
|
||||
self._shutdown.wait(timeout=self.interval_seconds)
|
||||
if not self._shutdown.is_set():
|
||||
self._take_snapshot()
|
||||
|
||||
def _take_snapshot(self) -> None:
|
||||
try:
|
||||
cpu_percent = psutil.cpu_percent(interval=0.1)
|
||||
memory = psutil.virtual_memory()
|
||||
disk = psutil.disk_usage(str(self.storage_root))
|
||||
|
||||
storage_bytes = 0
|
||||
with self._lock:
|
||||
storage = self._storage_ref
|
||||
if storage:
|
||||
try:
|
||||
buckets = storage.list_buckets()
|
||||
for bucket in buckets:
|
||||
stats = storage.bucket_stats(bucket.name, cache_ttl=60)
|
||||
storage_bytes += stats.get("total_bytes", stats.get("bytes", 0))
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to collect bucket stats: {e}")
|
||||
|
||||
snapshot = SystemMetricsSnapshot(
|
||||
timestamp=datetime.now(timezone.utc),
|
||||
cpu_percent=cpu_percent,
|
||||
memory_percent=memory.percent,
|
||||
disk_percent=disk.percent,
|
||||
storage_bytes=storage_bytes,
|
||||
)
|
||||
|
||||
with self._lock:
|
||||
self._snapshots.append(snapshot)
|
||||
self._prune_old_snapshots()
|
||||
self._save_history()
|
||||
|
||||
logger.debug(f"System metrics snapshot taken: CPU={cpu_percent:.1f}%, Memory={memory.percent:.1f}%")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to take system metrics snapshot: {e}")
|
||||
|
||||
def get_current(self) -> Dict[str, Any]:
|
||||
cpu_percent = psutil.cpu_percent(interval=0.1)
|
||||
memory = psutil.virtual_memory()
|
||||
disk = psutil.disk_usage(str(self.storage_root))
|
||||
boot_time = psutil.boot_time()
|
||||
uptime_seconds = time.time() - boot_time
|
||||
uptime_days = int(uptime_seconds / 86400)
|
||||
|
||||
total_buckets = 0
|
||||
total_objects = 0
|
||||
total_bytes_used = 0
|
||||
total_versions = 0
|
||||
|
||||
with self._lock:
|
||||
storage = self._storage_ref
|
||||
if storage:
|
||||
try:
|
||||
buckets = storage.list_buckets()
|
||||
total_buckets = len(buckets)
|
||||
for bucket in buckets:
|
||||
stats = storage.bucket_stats(bucket.name, cache_ttl=60)
|
||||
total_objects += stats.get("total_objects", stats.get("objects", 0))
|
||||
total_bytes_used += stats.get("total_bytes", stats.get("bytes", 0))
|
||||
total_versions += stats.get("version_count", 0)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to collect current bucket stats: {e}")
|
||||
|
||||
return {
|
||||
"cpu_percent": round(cpu_percent, 2),
|
||||
"memory": {
|
||||
"total": memory.total,
|
||||
"available": memory.available,
|
||||
"used": memory.used,
|
||||
"percent": round(memory.percent, 2),
|
||||
},
|
||||
"disk": {
|
||||
"total": disk.total,
|
||||
"free": disk.free,
|
||||
"used": disk.used,
|
||||
"percent": round(disk.percent, 2),
|
||||
},
|
||||
"app": {
|
||||
"buckets": total_buckets,
|
||||
"objects": total_objects,
|
||||
"versions": total_versions,
|
||||
"storage_bytes": total_bytes_used,
|
||||
"uptime_days": uptime_days,
|
||||
},
|
||||
}
|
||||
|
||||
def get_history(self, hours: Optional[int] = None) -> List[Dict[str, Any]]:
|
||||
with self._lock:
|
||||
snapshots = list(self._snapshots)
|
||||
|
||||
if hours:
|
||||
cutoff = datetime.now(timezone.utc).timestamp() - (hours * 3600)
|
||||
snapshots = [s for s in snapshots if s.timestamp.timestamp() > cutoff]
|
||||
|
||||
return [s.to_dict() for s in snapshots]
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self._shutdown.set()
|
||||
self._take_snapshot()
|
||||
self._snapshot_thread.join(timeout=5.0)
|
||||
@@ -1,6 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
APP_VERSION = "0.2.0"
|
||||
APP_VERSION = "0.3.1"
|
||||
|
||||
|
||||
def get_version() -> str:
|
||||
|
||||
108
app/website_domains.py
Normal file
108
app/website_domains.py
Normal file
@@ -0,0 +1,108 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
_DOMAIN_RE = re.compile(
|
||||
r"^(?!-)[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)*$"
|
||||
)
|
||||
|
||||
|
||||
def normalize_domain(raw: str) -> str:
|
||||
raw = raw.strip().lower()
|
||||
for prefix in ("https://", "http://"):
|
||||
if raw.startswith(prefix):
|
||||
raw = raw[len(prefix):]
|
||||
raw = raw.split("/", 1)[0]
|
||||
raw = raw.split("?", 1)[0]
|
||||
raw = raw.split("#", 1)[0]
|
||||
if ":" in raw:
|
||||
raw = raw.rsplit(":", 1)[0]
|
||||
return raw
|
||||
|
||||
|
||||
def is_valid_domain(domain: str) -> bool:
|
||||
if not domain or len(domain) > 253:
|
||||
return False
|
||||
return bool(_DOMAIN_RE.match(domain))
|
||||
|
||||
|
||||
class WebsiteDomainStore:
|
||||
def __init__(self, config_path: Path) -> None:
|
||||
self.config_path = config_path
|
||||
self._lock = threading.Lock()
|
||||
self._domains: Dict[str, str] = {}
|
||||
self._last_mtime: float = 0.0
|
||||
self.reload()
|
||||
|
||||
def reload(self) -> None:
|
||||
if not self.config_path.exists():
|
||||
self._domains = {}
|
||||
self._last_mtime = 0.0
|
||||
return
|
||||
try:
|
||||
self._last_mtime = self.config_path.stat().st_mtime
|
||||
with open(self.config_path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
if isinstance(data, dict):
|
||||
self._domains = {k.lower(): v for k, v in data.items()}
|
||||
else:
|
||||
self._domains = {}
|
||||
except (OSError, json.JSONDecodeError):
|
||||
self._domains = {}
|
||||
|
||||
def _maybe_reload(self) -> None:
|
||||
try:
|
||||
if self.config_path.exists():
|
||||
mtime = self.config_path.stat().st_mtime
|
||||
if mtime != self._last_mtime:
|
||||
self._last_mtime = mtime
|
||||
with open(self.config_path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
if isinstance(data, dict):
|
||||
self._domains = {k.lower(): v for k, v in data.items()}
|
||||
else:
|
||||
self._domains = {}
|
||||
elif self._domains:
|
||||
self._domains = {}
|
||||
self._last_mtime = 0.0
|
||||
except (OSError, json.JSONDecodeError):
|
||||
pass
|
||||
|
||||
def _save(self) -> None:
|
||||
self.config_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(self.config_path, "w", encoding="utf-8") as f:
|
||||
json.dump(self._domains, f, indent=2)
|
||||
self._last_mtime = self.config_path.stat().st_mtime
|
||||
|
||||
def list_all(self) -> List[Dict[str, str]]:
|
||||
with self._lock:
|
||||
self._maybe_reload()
|
||||
return [{"domain": d, "bucket": b} for d, b in self._domains.items()]
|
||||
|
||||
def get_bucket(self, domain: str) -> Optional[str]:
|
||||
with self._lock:
|
||||
self._maybe_reload()
|
||||
return self._domains.get(domain.lower())
|
||||
|
||||
def get_domains_for_bucket(self, bucket: str) -> List[str]:
|
||||
with self._lock:
|
||||
self._maybe_reload()
|
||||
return [d for d, b in self._domains.items() if b == bucket]
|
||||
|
||||
def set_mapping(self, domain: str, bucket: str) -> None:
|
||||
with self._lock:
|
||||
self._domains[domain.lower()] = bucket
|
||||
self._save()
|
||||
|
||||
def delete_mapping(self, domain: str) -> bool:
|
||||
with self._lock:
|
||||
key = domain.lower()
|
||||
if key not in self._domains:
|
||||
return False
|
||||
del self._domains[key]
|
||||
self._save()
|
||||
return True
|
||||
21
myfsio_core/Cargo.toml
Normal file
21
myfsio_core/Cargo.toml
Normal file
@@ -0,0 +1,21 @@
|
||||
[package]
|
||||
name = "myfsio_core"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
[lib]
|
||||
name = "myfsio_core"
|
||||
crate-type = ["cdylib"]
|
||||
|
||||
[dependencies]
|
||||
pyo3 = { version = "0.28", features = ["extension-module"] }
|
||||
hmac = "0.12"
|
||||
sha2 = "0.10"
|
||||
md-5 = "0.10"
|
||||
hex = "0.4"
|
||||
unicode-normalization = "0.1"
|
||||
serde_json = "1"
|
||||
regex = "1"
|
||||
lru = "0.14"
|
||||
parking_lot = "0.12"
|
||||
percent-encoding = "2"
|
||||
11
myfsio_core/pyproject.toml
Normal file
11
myfsio_core/pyproject.toml
Normal file
@@ -0,0 +1,11 @@
|
||||
[build-system]
|
||||
requires = ["maturin>=1.0,<2.0"]
|
||||
build-backend = "maturin"
|
||||
|
||||
[project]
|
||||
name = "myfsio_core"
|
||||
version = "0.1.0"
|
||||
requires-python = ">=3.10"
|
||||
|
||||
[tool.maturin]
|
||||
features = ["pyo3/extension-module"]
|
||||
90
myfsio_core/src/hashing.rs
Normal file
90
myfsio_core/src/hashing.rs
Normal file
@@ -0,0 +1,90 @@
|
||||
use md5::{Digest, Md5};
|
||||
use pyo3::exceptions::PyIOError;
|
||||
use pyo3::prelude::*;
|
||||
use sha2::Sha256;
|
||||
use std::fs::File;
|
||||
use std::io::Read;
|
||||
|
||||
const CHUNK_SIZE: usize = 65536;
|
||||
|
||||
#[pyfunction]
|
||||
pub fn md5_file(py: Python<'_>, path: &str) -> PyResult<String> {
|
||||
let path = path.to_owned();
|
||||
py.detach(move || {
|
||||
let mut file = File::open(&path)
|
||||
.map_err(|e| PyIOError::new_err(format!("Failed to open file: {}", e)))?;
|
||||
let mut hasher = Md5::new();
|
||||
let mut buf = vec![0u8; CHUNK_SIZE];
|
||||
loop {
|
||||
let n = file
|
||||
.read(&mut buf)
|
||||
.map_err(|e| PyIOError::new_err(format!("Failed to read file: {}", e)))?;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
hasher.update(&buf[..n]);
|
||||
}
|
||||
Ok(format!("{:x}", hasher.finalize()))
|
||||
})
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
pub fn md5_bytes(data: &[u8]) -> String {
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(data);
|
||||
format!("{:x}", hasher.finalize())
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
pub fn sha256_file(py: Python<'_>, path: &str) -> PyResult<String> {
|
||||
let path = path.to_owned();
|
||||
py.detach(move || {
|
||||
let mut file = File::open(&path)
|
||||
.map_err(|e| PyIOError::new_err(format!("Failed to open file: {}", e)))?;
|
||||
let mut hasher = Sha256::new();
|
||||
let mut buf = vec![0u8; CHUNK_SIZE];
|
||||
loop {
|
||||
let n = file
|
||||
.read(&mut buf)
|
||||
.map_err(|e| PyIOError::new_err(format!("Failed to read file: {}", e)))?;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
hasher.update(&buf[..n]);
|
||||
}
|
||||
Ok(format!("{:x}", hasher.finalize()))
|
||||
})
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
pub fn sha256_bytes(data: &[u8]) -> String {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(data);
|
||||
format!("{:x}", hasher.finalize())
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
pub fn md5_sha256_file(py: Python<'_>, path: &str) -> PyResult<(String, String)> {
|
||||
let path = path.to_owned();
|
||||
py.detach(move || {
|
||||
let mut file = File::open(&path)
|
||||
.map_err(|e| PyIOError::new_err(format!("Failed to open file: {}", e)))?;
|
||||
let mut md5_hasher = Md5::new();
|
||||
let mut sha_hasher = Sha256::new();
|
||||
let mut buf = vec![0u8; CHUNK_SIZE];
|
||||
loop {
|
||||
let n = file
|
||||
.read(&mut buf)
|
||||
.map_err(|e| PyIOError::new_err(format!("Failed to read file: {}", e)))?;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
md5_hasher.update(&buf[..n]);
|
||||
sha_hasher.update(&buf[..n]);
|
||||
}
|
||||
Ok((
|
||||
format!("{:x}", md5_hasher.finalize()),
|
||||
format!("{:x}", sha_hasher.finalize()),
|
||||
))
|
||||
})
|
||||
}
|
||||
34
myfsio_core/src/lib.rs
Normal file
34
myfsio_core/src/lib.rs
Normal file
@@ -0,0 +1,34 @@
|
||||
mod hashing;
|
||||
mod metadata;
|
||||
mod sigv4;
|
||||
mod validation;
|
||||
|
||||
use pyo3::prelude::*;
|
||||
|
||||
#[pymodule]
|
||||
mod myfsio_core {
|
||||
use super::*;
|
||||
|
||||
#[pymodule_init]
|
||||
fn init(m: &Bound<'_, PyModule>) -> PyResult<()> {
|
||||
m.add_function(wrap_pyfunction!(sigv4::verify_sigv4_signature, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(sigv4::derive_signing_key, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(sigv4::compute_signature, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(sigv4::build_string_to_sign, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(sigv4::constant_time_compare, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(sigv4::clear_signing_key_cache, m)?)?;
|
||||
|
||||
m.add_function(wrap_pyfunction!(hashing::md5_file, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(hashing::md5_bytes, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(hashing::sha256_file, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(hashing::sha256_bytes, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(hashing::md5_sha256_file, m)?)?;
|
||||
|
||||
m.add_function(wrap_pyfunction!(validation::validate_object_key, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(validation::validate_bucket_name, m)?)?;
|
||||
|
||||
m.add_function(wrap_pyfunction!(metadata::read_index_entry, m)?)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
71
myfsio_core/src/metadata.rs
Normal file
71
myfsio_core/src/metadata.rs
Normal file
@@ -0,0 +1,71 @@
|
||||
use pyo3::exceptions::PyValueError;
|
||||
use pyo3::prelude::*;
|
||||
use pyo3::types::{PyDict, PyList, PyString};
|
||||
use serde_json::Value;
|
||||
use std::fs;
|
||||
|
||||
const MAX_DEPTH: u32 = 64;
|
||||
|
||||
fn value_to_py(py: Python<'_>, v: &Value, depth: u32) -> PyResult<Py<PyAny>> {
|
||||
if depth > MAX_DEPTH {
|
||||
return Err(PyValueError::new_err("JSON nesting too deep"));
|
||||
}
|
||||
match v {
|
||||
Value::Null => Ok(py.None()),
|
||||
Value::Bool(b) => Ok((*b).into_pyobject(py)?.to_owned().into_any().unbind()),
|
||||
Value::Number(n) => {
|
||||
if let Some(i) = n.as_i64() {
|
||||
Ok(i.into_pyobject(py)?.into_any().unbind())
|
||||
} else if let Some(f) = n.as_f64() {
|
||||
Ok(f.into_pyobject(py)?.into_any().unbind())
|
||||
} else {
|
||||
Ok(py.None())
|
||||
}
|
||||
}
|
||||
Value::String(s) => Ok(PyString::new(py, s).into_any().unbind()),
|
||||
Value::Array(arr) => {
|
||||
let list = PyList::empty(py);
|
||||
for item in arr {
|
||||
list.append(value_to_py(py, item, depth + 1)?)?;
|
||||
}
|
||||
Ok(list.into_any().unbind())
|
||||
}
|
||||
Value::Object(map) => {
|
||||
let dict = PyDict::new(py);
|
||||
for (k, val) in map {
|
||||
dict.set_item(k, value_to_py(py, val, depth + 1)?)?;
|
||||
}
|
||||
Ok(dict.into_any().unbind())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
pub fn read_index_entry(
|
||||
py: Python<'_>,
|
||||
path: &str,
|
||||
entry_name: &str,
|
||||
) -> PyResult<Option<Py<PyAny>>> {
|
||||
let path_owned = path.to_owned();
|
||||
let entry_owned = entry_name.to_owned();
|
||||
|
||||
let entry: Option<Value> = py.detach(move || -> PyResult<Option<Value>> {
|
||||
let content = match fs::read_to_string(&path_owned) {
|
||||
Ok(c) => c,
|
||||
Err(_) => return Ok(None),
|
||||
};
|
||||
let parsed: Value = match serde_json::from_str(&content) {
|
||||
Ok(v) => v,
|
||||
Err(_) => return Ok(None),
|
||||
};
|
||||
match parsed {
|
||||
Value::Object(mut map) => Ok(map.remove(&entry_owned)),
|
||||
_ => Ok(None),
|
||||
}
|
||||
})?;
|
||||
|
||||
match entry {
|
||||
Some(val) => Ok(Some(value_to_py(py, &val, 0)?)),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
193
myfsio_core/src/sigv4.rs
Normal file
193
myfsio_core/src/sigv4.rs
Normal file
@@ -0,0 +1,193 @@
|
||||
use hmac::{Hmac, Mac};
|
||||
use lru::LruCache;
|
||||
use parking_lot::Mutex;
|
||||
use percent_encoding::{percent_encode, AsciiSet, NON_ALPHANUMERIC};
|
||||
use pyo3::prelude::*;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::num::NonZeroUsize;
|
||||
use std::sync::LazyLock;
|
||||
use std::time::Instant;
|
||||
|
||||
type HmacSha256 = Hmac<Sha256>;
|
||||
|
||||
struct CacheEntry {
|
||||
key: Vec<u8>,
|
||||
created: Instant,
|
||||
}
|
||||
|
||||
static SIGNING_KEY_CACHE: LazyLock<Mutex<LruCache<(String, String, String, String), CacheEntry>>> =
|
||||
LazyLock::new(|| Mutex::new(LruCache::new(NonZeroUsize::new(256).unwrap())));
|
||||
|
||||
const CACHE_TTL_SECS: u64 = 60;
|
||||
|
||||
const AWS_ENCODE_SET: &AsciiSet = &NON_ALPHANUMERIC
|
||||
.remove(b'-')
|
||||
.remove(b'_')
|
||||
.remove(b'.')
|
||||
.remove(b'~');
|
||||
|
||||
fn hmac_sha256(key: &[u8], msg: &[u8]) -> Vec<u8> {
|
||||
let mut mac = HmacSha256::new_from_slice(key).expect("HMAC key length is always valid");
|
||||
mac.update(msg);
|
||||
mac.finalize().into_bytes().to_vec()
|
||||
}
|
||||
|
||||
fn sha256_hex(data: &[u8]) -> String {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(data);
|
||||
hex::encode(hasher.finalize())
|
||||
}
|
||||
|
||||
fn aws_uri_encode(input: &str) -> String {
|
||||
percent_encode(input.as_bytes(), AWS_ENCODE_SET).to_string()
|
||||
}
|
||||
|
||||
fn derive_signing_key_cached(
|
||||
secret_key: &str,
|
||||
date_stamp: &str,
|
||||
region: &str,
|
||||
service: &str,
|
||||
) -> Vec<u8> {
|
||||
let cache_key = (
|
||||
secret_key.to_owned(),
|
||||
date_stamp.to_owned(),
|
||||
region.to_owned(),
|
||||
service.to_owned(),
|
||||
);
|
||||
|
||||
{
|
||||
let mut cache = SIGNING_KEY_CACHE.lock();
|
||||
if let Some(entry) = cache.get(&cache_key) {
|
||||
if entry.created.elapsed().as_secs() < CACHE_TTL_SECS {
|
||||
return entry.key.clone();
|
||||
}
|
||||
cache.pop(&cache_key);
|
||||
}
|
||||
}
|
||||
|
||||
let k_date = hmac_sha256(format!("AWS4{}", secret_key).as_bytes(), date_stamp.as_bytes());
|
||||
let k_region = hmac_sha256(&k_date, region.as_bytes());
|
||||
let k_service = hmac_sha256(&k_region, service.as_bytes());
|
||||
let k_signing = hmac_sha256(&k_service, b"aws4_request");
|
||||
|
||||
{
|
||||
let mut cache = SIGNING_KEY_CACHE.lock();
|
||||
cache.put(
|
||||
cache_key,
|
||||
CacheEntry {
|
||||
key: k_signing.clone(),
|
||||
created: Instant::now(),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
k_signing
|
||||
}
|
||||
|
||||
fn constant_time_compare_inner(a: &[u8], b: &[u8]) -> bool {
|
||||
if a.len() != b.len() {
|
||||
return false;
|
||||
}
|
||||
let mut result: u8 = 0;
|
||||
for (x, y) in a.iter().zip(b.iter()) {
|
||||
result |= x ^ y;
|
||||
}
|
||||
result == 0
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
pub fn verify_sigv4_signature(
|
||||
method: &str,
|
||||
canonical_uri: &str,
|
||||
query_params: Vec<(String, String)>,
|
||||
signed_headers_str: &str,
|
||||
header_values: Vec<(String, String)>,
|
||||
payload_hash: &str,
|
||||
amz_date: &str,
|
||||
date_stamp: &str,
|
||||
region: &str,
|
||||
service: &str,
|
||||
secret_key: &str,
|
||||
provided_signature: &str,
|
||||
) -> bool {
|
||||
let mut sorted_params = query_params;
|
||||
sorted_params.sort_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.cmp(&b.1)));
|
||||
|
||||
let canonical_query_string = sorted_params
|
||||
.iter()
|
||||
.map(|(k, v)| format!("{}={}", aws_uri_encode(k), aws_uri_encode(v)))
|
||||
.collect::<Vec<_>>()
|
||||
.join("&");
|
||||
|
||||
let mut canonical_headers = String::new();
|
||||
for (name, value) in &header_values {
|
||||
let lower_name = name.to_lowercase();
|
||||
let normalized = value.split_whitespace().collect::<Vec<_>>().join(" ");
|
||||
let final_value = if lower_name == "expect" && normalized.is_empty() {
|
||||
"100-continue"
|
||||
} else {
|
||||
&normalized
|
||||
};
|
||||
canonical_headers.push_str(&lower_name);
|
||||
canonical_headers.push(':');
|
||||
canonical_headers.push_str(final_value);
|
||||
canonical_headers.push('\n');
|
||||
}
|
||||
|
||||
let canonical_request = format!(
|
||||
"{}\n{}\n{}\n{}\n{}\n{}",
|
||||
method, canonical_uri, canonical_query_string, canonical_headers, signed_headers_str, payload_hash
|
||||
);
|
||||
|
||||
let credential_scope = format!("{}/{}/{}/aws4_request", date_stamp, region, service);
|
||||
let cr_hash = sha256_hex(canonical_request.as_bytes());
|
||||
let string_to_sign = format!(
|
||||
"AWS4-HMAC-SHA256\n{}\n{}\n{}",
|
||||
amz_date, credential_scope, cr_hash
|
||||
);
|
||||
|
||||
let signing_key = derive_signing_key_cached(secret_key, date_stamp, region, service);
|
||||
let calculated = hmac_sha256(&signing_key, string_to_sign.as_bytes());
|
||||
let calculated_hex = hex::encode(&calculated);
|
||||
|
||||
constant_time_compare_inner(calculated_hex.as_bytes(), provided_signature.as_bytes())
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
pub fn derive_signing_key(
|
||||
secret_key: &str,
|
||||
date_stamp: &str,
|
||||
region: &str,
|
||||
service: &str,
|
||||
) -> Vec<u8> {
|
||||
derive_signing_key_cached(secret_key, date_stamp, region, service)
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
pub fn compute_signature(signing_key: &[u8], string_to_sign: &str) -> String {
|
||||
let sig = hmac_sha256(signing_key, string_to_sign.as_bytes());
|
||||
hex::encode(sig)
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
pub fn build_string_to_sign(
|
||||
amz_date: &str,
|
||||
credential_scope: &str,
|
||||
canonical_request: &str,
|
||||
) -> String {
|
||||
let cr_hash = sha256_hex(canonical_request.as_bytes());
|
||||
format!(
|
||||
"AWS4-HMAC-SHA256\n{}\n{}\n{}",
|
||||
amz_date, credential_scope, cr_hash
|
||||
)
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
pub fn constant_time_compare(a: &str, b: &str) -> bool {
|
||||
constant_time_compare_inner(a.as_bytes(), b.as_bytes())
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
pub fn clear_signing_key_cache() {
|
||||
SIGNING_KEY_CACHE.lock().clear();
|
||||
}
|
||||
149
myfsio_core/src/validation.rs
Normal file
149
myfsio_core/src/validation.rs
Normal file
@@ -0,0 +1,149 @@
|
||||
use pyo3::prelude::*;
|
||||
use std::sync::LazyLock;
|
||||
use unicode_normalization::UnicodeNormalization;
|
||||
|
||||
const WINDOWS_RESERVED: &[&str] = &[
|
||||
"CON", "PRN", "AUX", "NUL", "COM0", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7",
|
||||
"COM8", "COM9", "LPT0", "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8",
|
||||
"LPT9",
|
||||
];
|
||||
|
||||
const WINDOWS_ILLEGAL_CHARS: &[char] = &['<', '>', ':', '"', '/', '\\', '|', '?', '*'];
|
||||
|
||||
const INTERNAL_FOLDERS: &[&str] = &[".meta", ".versions", ".multipart"];
|
||||
const SYSTEM_ROOT: &str = ".myfsio.sys";
|
||||
|
||||
static IP_REGEX: LazyLock<regex::Regex> =
|
||||
LazyLock::new(|| regex::Regex::new(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$").unwrap());
|
||||
|
||||
#[pyfunction]
|
||||
#[pyo3(signature = (object_key, max_length_bytes=1024, is_windows=false, reserved_prefixes=None))]
|
||||
pub fn validate_object_key(
|
||||
object_key: &str,
|
||||
max_length_bytes: usize,
|
||||
is_windows: bool,
|
||||
reserved_prefixes: Option<Vec<String>>,
|
||||
) -> PyResult<Option<String>> {
|
||||
if object_key.is_empty() {
|
||||
return Ok(Some("Object key required".to_string()));
|
||||
}
|
||||
|
||||
if object_key.contains('\0') {
|
||||
return Ok(Some("Object key contains null bytes".to_string()));
|
||||
}
|
||||
|
||||
let normalized: String = object_key.nfc().collect();
|
||||
|
||||
if normalized.len() > max_length_bytes {
|
||||
return Ok(Some(format!(
|
||||
"Object key exceeds maximum length of {} bytes",
|
||||
max_length_bytes
|
||||
)));
|
||||
}
|
||||
|
||||
if normalized.starts_with('/') || normalized.starts_with('\\') {
|
||||
return Ok(Some("Object key cannot start with a slash".to_string()));
|
||||
}
|
||||
|
||||
let parts: Vec<&str> = if cfg!(windows) || is_windows {
|
||||
normalized.split(['/', '\\']).collect()
|
||||
} else {
|
||||
normalized.split('/').collect()
|
||||
};
|
||||
|
||||
for part in &parts {
|
||||
if part.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
if *part == ".." {
|
||||
return Ok(Some(
|
||||
"Object key contains parent directory references".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
if *part == "." {
|
||||
return Ok(Some("Object key contains invalid segments".to_string()));
|
||||
}
|
||||
|
||||
if part.chars().any(|c| (c as u32) < 32) {
|
||||
return Ok(Some(
|
||||
"Object key contains control characters".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
if is_windows {
|
||||
if part.chars().any(|c| WINDOWS_ILLEGAL_CHARS.contains(&c)) {
|
||||
return Ok(Some(
|
||||
"Object key contains characters not supported on Windows filesystems"
|
||||
.to_string(),
|
||||
));
|
||||
}
|
||||
if part.ends_with(' ') || part.ends_with('.') {
|
||||
return Ok(Some(
|
||||
"Object key segments cannot end with spaces or periods on Windows".to_string(),
|
||||
));
|
||||
}
|
||||
let trimmed = part.trim_end_matches(['.', ' ']).to_uppercase();
|
||||
if WINDOWS_RESERVED.contains(&trimmed.as_str()) {
|
||||
return Ok(Some(format!("Invalid filename segment: {}", part)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let non_empty_parts: Vec<&str> = parts.iter().filter(|p| !p.is_empty()).copied().collect();
|
||||
if let Some(top) = non_empty_parts.first() {
|
||||
if INTERNAL_FOLDERS.contains(top) || *top == SYSTEM_ROOT {
|
||||
return Ok(Some("Object key uses a reserved prefix".to_string()));
|
||||
}
|
||||
|
||||
if let Some(ref prefixes) = reserved_prefixes {
|
||||
for prefix in prefixes {
|
||||
if *top == prefix.as_str() {
|
||||
return Ok(Some("Object key uses a reserved prefix".to_string()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
pub fn validate_bucket_name(bucket_name: &str) -> Option<String> {
|
||||
let len = bucket_name.len();
|
||||
if len < 3 || len > 63 {
|
||||
return Some("Bucket name must be between 3 and 63 characters".to_string());
|
||||
}
|
||||
|
||||
let bytes = bucket_name.as_bytes();
|
||||
if !bytes[0].is_ascii_lowercase() && !bytes[0].is_ascii_digit() {
|
||||
return Some(
|
||||
"Bucket name must start and end with a lowercase letter or digit".to_string(),
|
||||
);
|
||||
}
|
||||
if !bytes[len - 1].is_ascii_lowercase() && !bytes[len - 1].is_ascii_digit() {
|
||||
return Some(
|
||||
"Bucket name must start and end with a lowercase letter or digit".to_string(),
|
||||
);
|
||||
}
|
||||
|
||||
for &b in bytes {
|
||||
if !b.is_ascii_lowercase() && !b.is_ascii_digit() && b != b'.' && b != b'-' {
|
||||
return Some(
|
||||
"Bucket name can only contain lowercase letters, digits, dots, and hyphens"
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if bucket_name.contains("..") {
|
||||
return Some("Bucket name must not contain consecutive periods".to_string());
|
||||
}
|
||||
|
||||
if IP_REGEX.is_match(bucket_name) {
|
||||
return Some("Bucket name must not be formatted as an IP address".to_string());
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
@@ -2,9 +2,12 @@ Flask>=3.1.2
|
||||
Flask-Limiter>=4.1.1
|
||||
Flask-Cors>=6.0.2
|
||||
Flask-WTF>=1.2.2
|
||||
python-dotenv>=1.2.1
|
||||
pytest>=9.0.2
|
||||
requests>=2.32.5
|
||||
boto3>=1.42.14
|
||||
waitress>=3.0.2
|
||||
psutil>=7.1.3
|
||||
cryptography>=46.0.3
|
||||
cryptography>=46.0.3
|
||||
defusedxml>=0.7.1
|
||||
duckdb>=1.4.4
|
||||
72
run.py
72
run.py
@@ -5,7 +5,21 @@ import argparse
|
||||
import os
|
||||
import sys
|
||||
import warnings
|
||||
import multiprocessing
|
||||
from multiprocessing import Process
|
||||
from pathlib import Path
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
for _env_file in [
|
||||
Path("/opt/myfsio/myfsio.env"),
|
||||
Path.cwd() / ".env",
|
||||
Path.cwd() / "myfsio.env",
|
||||
]:
|
||||
if _env_file.exists():
|
||||
load_dotenv(_env_file, override=True)
|
||||
|
||||
from typing import Optional
|
||||
|
||||
from app import create_api_app, create_ui_app
|
||||
from app.config import AppConfig
|
||||
@@ -25,11 +39,23 @@ def _is_frozen() -> bool:
|
||||
return getattr(sys, 'frozen', False) or '__compiled__' in globals()
|
||||
|
||||
|
||||
def serve_api(port: int, prod: bool = False) -> None:
|
||||
def serve_api(port: int, prod: bool = False, config: Optional[AppConfig] = None) -> None:
|
||||
app = create_api_app()
|
||||
if prod:
|
||||
from waitress import serve
|
||||
serve(app, host=_server_host(), port=port, ident="MyFSIO")
|
||||
if config:
|
||||
serve(
|
||||
app,
|
||||
host=_server_host(),
|
||||
port=port,
|
||||
ident="MyFSIO",
|
||||
threads=config.server_threads,
|
||||
connection_limit=config.server_connection_limit,
|
||||
backlog=config.server_backlog,
|
||||
channel_timeout=config.server_channel_timeout,
|
||||
)
|
||||
else:
|
||||
serve(app, host=_server_host(), port=port, ident="MyFSIO")
|
||||
else:
|
||||
debug = _is_debug_enabled()
|
||||
if debug:
|
||||
@@ -37,11 +63,23 @@ def serve_api(port: int, prod: bool = False) -> None:
|
||||
app.run(host=_server_host(), port=port, debug=debug)
|
||||
|
||||
|
||||
def serve_ui(port: int, prod: bool = False) -> None:
|
||||
def serve_ui(port: int, prod: bool = False, config: Optional[AppConfig] = None) -> None:
|
||||
app = create_ui_app()
|
||||
if prod:
|
||||
from waitress import serve
|
||||
serve(app, host=_server_host(), port=port, ident="MyFSIO")
|
||||
if config:
|
||||
serve(
|
||||
app,
|
||||
host=_server_host(),
|
||||
port=port,
|
||||
ident="MyFSIO",
|
||||
threads=config.server_threads,
|
||||
connection_limit=config.server_connection_limit,
|
||||
backlog=config.server_backlog,
|
||||
channel_timeout=config.server_channel_timeout,
|
||||
)
|
||||
else:
|
||||
serve(app, host=_server_host(), port=port, ident="MyFSIO")
|
||||
else:
|
||||
debug = _is_debug_enabled()
|
||||
if debug:
|
||||
@@ -50,6 +88,10 @@ def serve_ui(port: int, prod: bool = False) -> None:
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
multiprocessing.freeze_support()
|
||||
if _is_frozen():
|
||||
multiprocessing.set_start_method("spawn", force=True)
|
||||
|
||||
parser = argparse.ArgumentParser(description="Run the S3 clone services.")
|
||||
parser.add_argument("--mode", choices=["api", "ui", "both"], default="both")
|
||||
parser.add_argument("--api-port", type=int, default=5000)
|
||||
@@ -60,7 +102,6 @@ if __name__ == "__main__":
|
||||
parser.add_argument("--show-config", action="store_true", help="Show configuration summary and exit")
|
||||
args = parser.parse_args()
|
||||
|
||||
# Handle config check/show modes
|
||||
if args.check_config or args.show_config:
|
||||
config = AppConfig.from_env()
|
||||
config.print_startup_summary()
|
||||
@@ -70,49 +111,50 @@ if __name__ == "__main__":
|
||||
sys.exit(1 if critical else 0)
|
||||
sys.exit(0)
|
||||
|
||||
# Default to production mode when running as compiled binary
|
||||
# unless --dev is explicitly passed
|
||||
prod_mode = args.prod or (_is_frozen() and not args.dev)
|
||||
|
||||
# Validate configuration before starting
|
||||
config = AppConfig.from_env()
|
||||
|
||||
# Show startup summary only on first run (when marker file doesn't exist)
|
||||
first_run_marker = config.storage_root / ".myfsio.sys" / ".initialized"
|
||||
is_first_run = not first_run_marker.exists()
|
||||
|
||||
if is_first_run:
|
||||
config.print_startup_summary()
|
||||
|
||||
# Check for critical issues that should prevent startup
|
||||
issues = config.validate_and_report()
|
||||
critical_issues = [i for i in issues if i.startswith("CRITICAL:")]
|
||||
if critical_issues:
|
||||
print("ABORTING: Critical configuration issues detected. Fix them before starting.")
|
||||
print("ABORTING: Critical configuration issues detected. Please fix them before starting.")
|
||||
sys.exit(1)
|
||||
|
||||
# Create the marker file to indicate successful first run
|
||||
try:
|
||||
first_run_marker.parent.mkdir(parents=True, exist_ok=True)
|
||||
first_run_marker.write_text(f"Initialized on {__import__('datetime').datetime.now().isoformat()}\n")
|
||||
except OSError:
|
||||
pass # Non-critical, just skip marker creation
|
||||
pass
|
||||
|
||||
if prod_mode:
|
||||
print("Running in production mode (Waitress)")
|
||||
issues = config.validate_and_report()
|
||||
critical_issues = [i for i in issues if i.startswith("CRITICAL:")]
|
||||
if critical_issues:
|
||||
for issue in critical_issues:
|
||||
print(f" {issue}")
|
||||
print("ABORTING: Critical configuration issues detected. Please fix them before starting.")
|
||||
sys.exit(1)
|
||||
else:
|
||||
print("Running in development mode (Flask dev server)")
|
||||
|
||||
if args.mode in {"api", "both"}:
|
||||
print(f"Starting API server on port {args.api_port}...")
|
||||
api_proc = Process(target=serve_api, args=(args.api_port, prod_mode), daemon=True)
|
||||
api_proc = Process(target=serve_api, args=(args.api_port, prod_mode, config), daemon=True)
|
||||
api_proc.start()
|
||||
else:
|
||||
api_proc = None
|
||||
|
||||
if args.mode in {"ui", "both"}:
|
||||
print(f"Starting UI server on port {args.ui_port}...")
|
||||
serve_ui(args.ui_port, prod_mode)
|
||||
serve_ui(args.ui_port, prod_mode, config)
|
||||
elif api_proc:
|
||||
try:
|
||||
api_proc.join()
|
||||
|
||||
@@ -192,31 +192,86 @@ cat > "$INSTALL_DIR/myfsio.env" << EOF
|
||||
# Generated by install.sh on $(date)
|
||||
# Documentation: https://go.jzwsite.com/myfsio
|
||||
|
||||
# Storage paths
|
||||
# =============================================================================
|
||||
# STORAGE PATHS
|
||||
# =============================================================================
|
||||
STORAGE_ROOT=$DATA_DIR
|
||||
LOG_DIR=$LOG_DIR
|
||||
|
||||
# Network
|
||||
# =============================================================================
|
||||
# NETWORK
|
||||
# =============================================================================
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=$API_PORT
|
||||
|
||||
# Security - CHANGE IN PRODUCTION
|
||||
SECRET_KEY=$SECRET_KEY
|
||||
CORS_ORIGINS=*
|
||||
|
||||
# Public URL (set this if behind a reverse proxy)
|
||||
# Public URL (set this if behind a reverse proxy for presigned URLs)
|
||||
$(if [[ -n "$API_URL" ]]; then echo "API_BASE_URL=$API_URL"; else echo "# API_BASE_URL=https://s3.example.com"; fi)
|
||||
|
||||
# Logging
|
||||
# =============================================================================
|
||||
# SECURITY
|
||||
# =============================================================================
|
||||
# Secret key for session signing (auto-generated if not set)
|
||||
SECRET_KEY=$SECRET_KEY
|
||||
|
||||
# CORS settings - restrict in production
|
||||
CORS_ORIGINS=*
|
||||
|
||||
# Brute-force protection
|
||||
AUTH_MAX_ATTEMPTS=5
|
||||
AUTH_LOCKOUT_MINUTES=15
|
||||
|
||||
# Reverse proxy settings (set to number of trusted proxies in front)
|
||||
# NUM_TRUSTED_PROXIES=1
|
||||
|
||||
# Allow internal admin endpoints (only enable on trusted networks)
|
||||
# ALLOW_INTERNAL_ENDPOINTS=false
|
||||
|
||||
# Allowed hosts for redirects (comma-separated, empty = restrict all)
|
||||
# ALLOWED_REDIRECT_HOSTS=
|
||||
|
||||
# =============================================================================
|
||||
# LOGGING
|
||||
# =============================================================================
|
||||
LOG_LEVEL=INFO
|
||||
LOG_TO_FILE=true
|
||||
|
||||
# Rate limiting
|
||||
# =============================================================================
|
||||
# RATE LIMITING
|
||||
# =============================================================================
|
||||
RATE_LIMIT_DEFAULT=200 per minute
|
||||
# RATE_LIMIT_LIST_BUCKETS=60 per minute
|
||||
# RATE_LIMIT_BUCKET_OPS=120 per minute
|
||||
# RATE_LIMIT_OBJECT_OPS=240 per minute
|
||||
# RATE_LIMIT_ADMIN=60 per minute
|
||||
|
||||
# Optional: Encryption (uncomment to enable)
|
||||
# =============================================================================
|
||||
# SERVER TUNING (0 = auto-detect based on system resources)
|
||||
# =============================================================================
|
||||
# SERVER_THREADS=0
|
||||
# SERVER_CONNECTION_LIMIT=0
|
||||
# SERVER_BACKLOG=0
|
||||
# SERVER_CHANNEL_TIMEOUT=120
|
||||
|
||||
# =============================================================================
|
||||
# ENCRYPTION (uncomment to enable)
|
||||
# =============================================================================
|
||||
# ENCRYPTION_ENABLED=true
|
||||
# KMS_ENABLED=true
|
||||
|
||||
# =============================================================================
|
||||
# SITE SYNC / REPLICATION (for multi-site deployments)
|
||||
# =============================================================================
|
||||
# SITE_ID=site-1
|
||||
# SITE_ENDPOINT=https://s3-site1.example.com
|
||||
# SITE_REGION=us-east-1
|
||||
# SITE_SYNC_ENABLED=false
|
||||
|
||||
# =============================================================================
|
||||
# OPTIONAL FEATURES
|
||||
# =============================================================================
|
||||
# LIFECYCLE_ENABLED=false
|
||||
# METRICS_HISTORY_ENABLED=false
|
||||
# OPERATION_METRICS_ENABLED=false
|
||||
EOF
|
||||
chmod 600 "$INSTALL_DIR/myfsio.env"
|
||||
echo " [OK] Created $INSTALL_DIR/myfsio.env"
|
||||
@@ -308,7 +363,7 @@ if [[ "$SKIP_SYSTEMD" != true ]]; then
|
||||
systemctl start myfsio
|
||||
echo " [OK] Service started"
|
||||
echo ""
|
||||
|
||||
|
||||
read -p "Would you like to enable MyFSIO to start on boot? [Y/n] " -n 1 -r
|
||||
echo
|
||||
if [[ ! $REPLY =~ ^[Nn]$ ]]; then
|
||||
@@ -316,12 +371,37 @@ if [[ "$SKIP_SYSTEMD" != true ]]; then
|
||||
echo " [OK] Service enabled on boot"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
sleep 2
|
||||
|
||||
echo " Waiting for service initialization..."
|
||||
sleep 3
|
||||
|
||||
echo " Service Status:"
|
||||
echo " ---------------"
|
||||
if systemctl is-active --quiet myfsio; then
|
||||
echo " [OK] MyFSIO is running"
|
||||
|
||||
IAM_FILE="$DATA_DIR/.myfsio.sys/config/iam.json"
|
||||
if [[ -f "$IAM_FILE" ]]; then
|
||||
echo ""
|
||||
echo " ============================================"
|
||||
echo " ADMIN CREDENTIALS (save these securely!)"
|
||||
echo " ============================================"
|
||||
if command -v jq &>/dev/null; then
|
||||
ACCESS_KEY=$(jq -r '.users[0].access_key' "$IAM_FILE" 2>/dev/null)
|
||||
SECRET_KEY=$(jq -r '.users[0].secret_key' "$IAM_FILE" 2>/dev/null)
|
||||
else
|
||||
ACCESS_KEY=$(grep -o '"access_key"[[:space:]]*:[[:space:]]*"[^"]*"' "$IAM_FILE" | head -1 | sed 's/.*"\([^"]*\)"$/\1/')
|
||||
SECRET_KEY=$(grep -o '"secret_key"[[:space:]]*:[[:space:]]*"[^"]*"' "$IAM_FILE" | head -1 | sed 's/.*"\([^"]*\)"$/\1/')
|
||||
fi
|
||||
if [[ -n "$ACCESS_KEY" && -n "$SECRET_KEY" ]]; then
|
||||
echo " Access Key: $ACCESS_KEY"
|
||||
echo " Secret Key: $SECRET_KEY"
|
||||
else
|
||||
echo " [!] Could not parse credentials from $IAM_FILE"
|
||||
echo " Check the file manually or view service logs."
|
||||
fi
|
||||
echo " ============================================"
|
||||
fi
|
||||
else
|
||||
echo " [WARNING] MyFSIO may not have started correctly"
|
||||
echo " Check logs with: journalctl -u myfsio -f"
|
||||
@@ -346,19 +426,26 @@ echo "Access Points:"
|
||||
echo " API: http://$(hostname -I 2>/dev/null | awk '{print $1}' || echo "localhost"):$API_PORT"
|
||||
echo " UI: http://$(hostname -I 2>/dev/null | awk '{print $1}' || echo "localhost"):$UI_PORT/ui"
|
||||
echo ""
|
||||
echo "Default Credentials:"
|
||||
echo " Username: localadmin"
|
||||
echo " Password: localadmin"
|
||||
echo " [!] WARNING: Change these immediately after first login!"
|
||||
echo "Credentials:"
|
||||
echo " Admin credentials were shown above (if service was started)."
|
||||
echo " You can also find them in: $DATA_DIR/.myfsio.sys/config/iam.json"
|
||||
echo ""
|
||||
echo "Configuration Files:"
|
||||
echo " Environment: $INSTALL_DIR/myfsio.env"
|
||||
echo " IAM Users: $DATA_DIR/.myfsio.sys/config/iam.json"
|
||||
echo " Bucket Policies: $DATA_DIR/.myfsio.sys/config/bucket_policies.json"
|
||||
echo " Secret Key: $DATA_DIR/.myfsio.sys/config/.secret (auto-generated)"
|
||||
echo ""
|
||||
echo "Security Notes:"
|
||||
echo " - Rate limiting is enabled by default (200 req/min)"
|
||||
echo " - Brute-force protection: 5 attempts, 15 min lockout"
|
||||
echo " - Set CORS_ORIGINS to specific domains in production"
|
||||
echo " - Set NUM_TRUSTED_PROXIES if behind a reverse proxy"
|
||||
echo ""
|
||||
echo "Useful Commands:"
|
||||
echo " Check status: sudo systemctl status myfsio"
|
||||
echo " View logs: sudo journalctl -u myfsio -f"
|
||||
echo " Validate config: $INSTALL_DIR/myfsio --check-config"
|
||||
echo " Restart: sudo systemctl restart myfsio"
|
||||
echo " Stop: sudo systemctl stop myfsio"
|
||||
echo ""
|
||||
|
||||
@@ -88,7 +88,8 @@ echo "The following items will be removed:"
|
||||
echo ""
|
||||
echo " Install directory: $INSTALL_DIR"
|
||||
if [[ "$KEEP_DATA" != true ]]; then
|
||||
echo " Data directory: $DATA_DIR (ALL YOUR DATA WILL BE DELETED!)"
|
||||
echo " Data directory: $DATA_DIR"
|
||||
echo " [!] ALL DATA, IAM USERS, AND ENCRYPTION KEYS WILL BE DELETED!"
|
||||
else
|
||||
echo " Data directory: $DATA_DIR (WILL BE KEPT)"
|
||||
fi
|
||||
@@ -227,8 +228,15 @@ echo ""
|
||||
if [[ "$KEEP_DATA" == true ]]; then
|
||||
echo "Your data has been preserved at: $DATA_DIR"
|
||||
echo ""
|
||||
echo "To reinstall MyFSIO with existing data, run:"
|
||||
echo " curl -fsSL https://go.jzwsite.com/myfsio-install | sudo bash"
|
||||
echo "Preserved files include:"
|
||||
echo " - All buckets and objects"
|
||||
echo " - IAM configuration: $DATA_DIR/.myfsio.sys/config/iam.json"
|
||||
echo " - Bucket policies: $DATA_DIR/.myfsio.sys/config/bucket_policies.json"
|
||||
echo " - Secret key: $DATA_DIR/.myfsio.sys/config/.secret"
|
||||
echo " - Encryption keys: $DATA_DIR/.myfsio.sys/keys/ (if encryption was enabled)"
|
||||
echo ""
|
||||
echo "To reinstall MyFSIO with existing data:"
|
||||
echo " ./install.sh --data-dir $DATA_DIR"
|
||||
echo ""
|
||||
fi
|
||||
|
||||
|
||||
@@ -1081,11 +1081,17 @@ html.sidebar-will-collapse .sidebar-user {
|
||||
letter-spacing: 0.08em;
|
||||
}
|
||||
|
||||
.main-content:has(.docs-sidebar) {
|
||||
overflow-x: visible;
|
||||
}
|
||||
|
||||
.docs-sidebar {
|
||||
position: sticky;
|
||||
top: 1.5rem;
|
||||
border-radius: 1rem;
|
||||
border: 1px solid var(--myfsio-card-border);
|
||||
max-height: calc(100vh - 3rem);
|
||||
overflow-y: auto;
|
||||
}
|
||||
|
||||
.docs-sidebar-callouts {
|
||||
@@ -1145,17 +1151,123 @@ html.sidebar-will-collapse .sidebar-user {
|
||||
}
|
||||
|
||||
.iam-user-card {
|
||||
border: 1px solid var(--myfsio-card-border);
|
||||
border-radius: 0.75rem;
|
||||
transition: box-shadow 0.2s ease, transform 0.2s ease;
|
||||
position: relative;
|
||||
border: 1px solid var(--myfsio-card-border) !important;
|
||||
border-radius: 1rem !important;
|
||||
overflow: hidden;
|
||||
transition: all 0.2s cubic-bezier(0.4, 0, 0.2, 1);
|
||||
}
|
||||
|
||||
.iam-user-card::before {
|
||||
content: '';
|
||||
position: absolute;
|
||||
top: 0;
|
||||
left: 0;
|
||||
right: 0;
|
||||
height: 4px;
|
||||
background: linear-gradient(90deg, #3b82f6, #8b5cf6);
|
||||
opacity: 0;
|
||||
transition: opacity 0.2s ease;
|
||||
}
|
||||
|
||||
.iam-user-card:hover {
|
||||
box-shadow: 0 4px 12px rgba(0, 0, 0, 0.1);
|
||||
transform: translateY(-2px);
|
||||
box-shadow: 0 8px 24px -4px rgba(0, 0, 0, 0.12), 0 4px 8px -4px rgba(0, 0, 0, 0.08);
|
||||
border-color: var(--myfsio-accent) !important;
|
||||
}
|
||||
|
||||
.iam-user-card:hover::before {
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
[data-theme='dark'] .iam-user-card:hover {
|
||||
box-shadow: 0 4px 12px rgba(0, 0, 0, 0.3);
|
||||
box-shadow: 0 8px 24px -4px rgba(0, 0, 0, 0.4), 0 4px 8px -4px rgba(0, 0, 0, 0.3);
|
||||
}
|
||||
|
||||
.iam-admin-card::before {
|
||||
background: linear-gradient(90deg, #f59e0b, #ef4444);
|
||||
}
|
||||
|
||||
.iam-role-badge {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
padding: 0.25em 0.65em;
|
||||
border-radius: 999px;
|
||||
font-size: 0.7rem;
|
||||
font-weight: 600;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.03em;
|
||||
}
|
||||
|
||||
.iam-role-admin {
|
||||
background: rgba(245, 158, 11, 0.15);
|
||||
color: #d97706;
|
||||
}
|
||||
|
||||
[data-theme='dark'] .iam-role-admin {
|
||||
background: rgba(245, 158, 11, 0.25);
|
||||
color: #fbbf24;
|
||||
}
|
||||
|
||||
.iam-role-user {
|
||||
background: rgba(59, 130, 246, 0.12);
|
||||
color: #2563eb;
|
||||
}
|
||||
|
||||
[data-theme='dark'] .iam-role-user {
|
||||
background: rgba(59, 130, 246, 0.2);
|
||||
color: #60a5fa;
|
||||
}
|
||||
|
||||
.iam-perm-badge {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 0.25rem;
|
||||
padding: 0.3em 0.6em;
|
||||
border-radius: 999px;
|
||||
font-size: 0.75rem;
|
||||
font-weight: 500;
|
||||
background: rgba(59, 130, 246, 0.08);
|
||||
color: var(--myfsio-text);
|
||||
border: 1px solid rgba(59, 130, 246, 0.15);
|
||||
}
|
||||
|
||||
[data-theme='dark'] .iam-perm-badge {
|
||||
background: rgba(59, 130, 246, 0.15);
|
||||
border-color: rgba(59, 130, 246, 0.25);
|
||||
}
|
||||
|
||||
.iam-copy-key {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
width: 22px;
|
||||
height: 22px;
|
||||
padding: 0;
|
||||
border: none;
|
||||
background: transparent;
|
||||
color: var(--myfsio-muted);
|
||||
border-radius: 4px;
|
||||
cursor: pointer;
|
||||
transition: all 0.15s ease;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.iam-copy-key:hover {
|
||||
background: var(--myfsio-hover-bg);
|
||||
color: var(--myfsio-text);
|
||||
}
|
||||
|
||||
.iam-no-results {
|
||||
text-align: center;
|
||||
padding: 2rem 1rem;
|
||||
color: var(--myfsio-muted);
|
||||
}
|
||||
|
||||
@media (max-width: 768px) {
|
||||
.iam-user-card:hover {
|
||||
transform: none;
|
||||
}
|
||||
}
|
||||
|
||||
.user-avatar-lg {
|
||||
@@ -1282,6 +1394,20 @@ html.sidebar-will-collapse .sidebar-user {
|
||||
padding: 2rem 1rem;
|
||||
}
|
||||
|
||||
#preview-text {
|
||||
padding: 1rem 1.125rem;
|
||||
max-height: 360px;
|
||||
overflow: auto;
|
||||
white-space: pre-wrap;
|
||||
word-break: break-word;
|
||||
font-family: 'SFMono-Regular', 'Menlo', 'Consolas', 'Liberation Mono', monospace;
|
||||
font-size: .8rem;
|
||||
line-height: 1.6;
|
||||
tab-size: 4;
|
||||
color: var(--myfsio-text);
|
||||
background: transparent;
|
||||
}
|
||||
|
||||
.upload-progress-stack {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
@@ -2799,6 +2925,112 @@ body:has(.login-card) .main-wrapper {
|
||||
padding-top: 0 !important;
|
||||
}
|
||||
|
||||
.context-menu {
|
||||
position: fixed;
|
||||
z-index: 1060;
|
||||
min-width: 180px;
|
||||
background: var(--myfsio-card-bg);
|
||||
border: 1px solid var(--myfsio-card-border);
|
||||
border-radius: 0.5rem;
|
||||
box-shadow: 0 10px 25px -5px rgba(0, 0, 0, 0.15), 0 8px 10px -6px rgba(0, 0, 0, 0.1);
|
||||
padding: 0.25rem 0;
|
||||
font-size: 0.875rem;
|
||||
}
|
||||
|
||||
[data-theme='dark'] .context-menu {
|
||||
box-shadow: 0 10px 25px -5px rgba(0, 0, 0, 0.4), 0 8px 10px -6px rgba(0, 0, 0, 0.3);
|
||||
}
|
||||
|
||||
.context-menu-item {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.625rem;
|
||||
padding: 0.5rem 0.875rem;
|
||||
color: var(--myfsio-text);
|
||||
cursor: pointer;
|
||||
transition: background-color 0.1s ease;
|
||||
border: none;
|
||||
background: none;
|
||||
width: 100%;
|
||||
text-align: left;
|
||||
font-size: inherit;
|
||||
}
|
||||
|
||||
.context-menu-item:hover {
|
||||
background-color: var(--myfsio-hover-bg);
|
||||
}
|
||||
|
||||
.context-menu-item.text-danger:hover {
|
||||
background-color: rgba(239, 68, 68, 0.1);
|
||||
}
|
||||
|
||||
.context-menu-divider {
|
||||
height: 1px;
|
||||
background: var(--myfsio-card-border);
|
||||
margin: 0.25rem 0;
|
||||
}
|
||||
|
||||
.context-menu-shortcut {
|
||||
margin-left: auto;
|
||||
font-size: 0.75rem;
|
||||
color: var(--myfsio-muted);
|
||||
}
|
||||
|
||||
.kbd-shortcuts-list {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0.5rem;
|
||||
}
|
||||
|
||||
.kbd-shortcuts-list .shortcut-row {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
padding: 0.375rem 0;
|
||||
}
|
||||
|
||||
.kbd-shortcuts-list kbd {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
min-width: 1.75rem;
|
||||
padding: 0.2rem 0.5rem;
|
||||
font-family: inherit;
|
||||
font-size: 0.75rem;
|
||||
font-weight: 600;
|
||||
background: var(--myfsio-preview-bg);
|
||||
border: 1px solid var(--myfsio-card-border);
|
||||
border-radius: 0.25rem;
|
||||
box-shadow: 0 1px 0 1px rgba(0, 0, 0, 0.05);
|
||||
color: var(--myfsio-text);
|
||||
}
|
||||
|
||||
[data-theme='dark'] .kbd-shortcuts-list kbd {
|
||||
background: rgba(255, 255, 255, 0.1);
|
||||
box-shadow: 0 1px 0 1px rgba(0, 0, 0, 0.2);
|
||||
}
|
||||
|
||||
.sort-dropdown .dropdown-item.active,
|
||||
.sort-dropdown .dropdown-item:active {
|
||||
background-color: var(--myfsio-hover-bg);
|
||||
color: var(--myfsio-text);
|
||||
}
|
||||
|
||||
.sort-dropdown .dropdown-item {
|
||||
font-size: 0.875rem;
|
||||
padding: 0.375rem 1rem;
|
||||
}
|
||||
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
*,
|
||||
*::before,
|
||||
*::after {
|
||||
animation-duration: 0.01ms !important;
|
||||
animation-iteration-count: 1 !important;
|
||||
transition-duration: 0.01ms !important;
|
||||
}
|
||||
}
|
||||
|
||||
@media print {
|
||||
.sidebar,
|
||||
.mobile-header {
|
||||
|
||||
4686
static/js/bucket-detail-main.js
Normal file
4686
static/js/bucket-detail-main.js
Normal file
File diff suppressed because it is too large
Load Diff
343
static/js/connections-management.js
Normal file
343
static/js/connections-management.js
Normal file
@@ -0,0 +1,343 @@
|
||||
window.ConnectionsManagement = (function() {
|
||||
'use strict';
|
||||
|
||||
var endpoints = {};
|
||||
var csrfToken = '';
|
||||
|
||||
function init(config) {
|
||||
endpoints = config.endpoints || {};
|
||||
csrfToken = config.csrfToken || '';
|
||||
|
||||
setupEventListeners();
|
||||
checkAllConnectionHealth();
|
||||
}
|
||||
|
||||
function togglePassword(id) {
|
||||
var input = document.getElementById(id);
|
||||
if (input) {
|
||||
input.type = input.type === 'password' ? 'text' : 'password';
|
||||
}
|
||||
}
|
||||
|
||||
async function testConnection(formId, resultId) {
|
||||
var form = document.getElementById(formId);
|
||||
var resultDiv = document.getElementById(resultId);
|
||||
if (!form || !resultDiv) return;
|
||||
|
||||
var formData = new FormData(form);
|
||||
var data = {};
|
||||
formData.forEach(function(value, key) {
|
||||
if (key !== 'csrf_token') {
|
||||
data[key] = value;
|
||||
}
|
||||
});
|
||||
|
||||
resultDiv.innerHTML = '<div class="text-info"><span class="spinner-border spinner-border-sm" role="status" aria-hidden="true"></span> Testing connection...</div>';
|
||||
|
||||
var controller = new AbortController();
|
||||
var timeoutId = setTimeout(function() { controller.abort(); }, 20000);
|
||||
|
||||
try {
|
||||
var response = await fetch(endpoints.test, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'X-CSRFToken': csrfToken
|
||||
},
|
||||
body: JSON.stringify(data),
|
||||
signal: controller.signal
|
||||
});
|
||||
clearTimeout(timeoutId);
|
||||
|
||||
var result = await response.json();
|
||||
if (response.ok) {
|
||||
resultDiv.innerHTML = '<div class="text-success">' +
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="me-1" viewBox="0 0 16 16">' +
|
||||
'<path d="M16 8A8 8 0 1 1 0 8a8 8 0 0 1 16 0zm-3.97-3.03a.75.75 0 0 0-1.08.022L7.477 9.417 5.384 7.323a.75.75 0 0 0-1.06 1.06L6.97 11.03a.75.75 0 0 0 1.079-.02l3.992-4.99a.75.75 0 0 0-.01-1.05z"/>' +
|
||||
'</svg>' + window.UICore.escapeHtml(result.message) + '</div>';
|
||||
} else {
|
||||
resultDiv.innerHTML = '<div class="text-danger">' +
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="me-1" viewBox="0 0 16 16">' +
|
||||
'<path d="M16 8A8 8 0 1 1 0 8a8 8 0 0 1 16 0zM5.354 4.646a.5.5 0 1 0-.708.708L7.293 8l-2.647 2.646a.5.5 0 0 0 .708.708L8 8.707l2.646 2.647a.5.5 0 0 0 .708-.708L8.707 8l2.647-2.646a.5.5 0 0 0-.708-.708L8 7.293 5.354 4.646z"/>' +
|
||||
'</svg>' + window.UICore.escapeHtml(result.message) + '</div>';
|
||||
}
|
||||
} catch (error) {
|
||||
clearTimeout(timeoutId);
|
||||
var message = error.name === 'AbortError'
|
||||
? 'Connection test timed out - endpoint may be unreachable'
|
||||
: 'Connection failed: Network error';
|
||||
resultDiv.innerHTML = '<div class="text-danger">' +
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="me-1" viewBox="0 0 16 16">' +
|
||||
'<path d="M16 8A8 8 0 1 1 0 8a8 8 0 0 1 16 0zM5.354 4.646a.5.5 0 1 0-.708.708L7.293 8l-2.647 2.646a.5.5 0 0 0 .708.708L8 8.707l2.646 2.647a.5.5 0 0 0 .708-.708L8.707 8l2.647-2.646a.5.5 0 0 0-.708-.708L8 7.293 5.354 4.646z"/>' +
|
||||
'</svg>' + message + '</div>';
|
||||
}
|
||||
}
|
||||
|
||||
async function checkConnectionHealth(connectionId, statusEl) {
|
||||
if (!statusEl) return;
|
||||
|
||||
try {
|
||||
var controller = new AbortController();
|
||||
var timeoutId = setTimeout(function() { controller.abort(); }, 10000);
|
||||
|
||||
var response = await fetch(endpoints.healthTemplate.replace('CONNECTION_ID', connectionId), {
|
||||
signal: controller.signal
|
||||
});
|
||||
clearTimeout(timeoutId);
|
||||
|
||||
var data = await response.json();
|
||||
if (data.healthy) {
|
||||
statusEl.innerHTML = '<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="text-success" viewBox="0 0 16 16">' +
|
||||
'<path d="M16 8A8 8 0 1 1 0 8a8 8 0 0 1 16 0zm-3.97-3.03a.75.75 0 0 0-1.08.022L7.477 9.417 5.384 7.323a.75.75 0 0 0-1.06 1.06L6.97 11.03a.75.75 0 0 0 1.079-.02l3.992-4.99a.75.75 0 0 0-.01-1.05z"/></svg>';
|
||||
statusEl.setAttribute('data-status', 'healthy');
|
||||
statusEl.setAttribute('title', 'Connected');
|
||||
} else {
|
||||
statusEl.innerHTML = '<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="text-danger" viewBox="0 0 16 16">' +
|
||||
'<path d="M16 8A8 8 0 1 1 0 8a8 8 0 0 1 16 0zM5.354 4.646a.5.5 0 1 0-.708.708L7.293 8l-2.647 2.646a.5.5 0 0 0 .708.708L8 8.707l2.646 2.647a.5.5 0 0 0 .708-.708L8.707 8l2.647-2.646a.5.5 0 0 0-.708-.708L8 7.293 5.354 4.646z"/></svg>';
|
||||
statusEl.setAttribute('data-status', 'unhealthy');
|
||||
statusEl.setAttribute('title', data.error || 'Unreachable');
|
||||
}
|
||||
} catch (error) {
|
||||
statusEl.innerHTML = '<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="text-warning" viewBox="0 0 16 16">' +
|
||||
'<path d="M8.982 1.566a1.13 1.13 0 0 0-1.96 0L.165 13.233c-.457.778.091 1.767.98 1.767h13.713c.889 0 1.438-.99.98-1.767L8.982 1.566zM8 5c.535 0 .954.462.9.995l-.35 3.507a.552.552 0 0 1-1.1 0L7.1 5.995A.905.905 0 0 1 8 5zm.002 6a1 1 0 1 1 0 2 1 1 0 0 1 0-2z"/></svg>';
|
||||
statusEl.setAttribute('data-status', 'unknown');
|
||||
statusEl.setAttribute('title', 'Could not check status');
|
||||
}
|
||||
}
|
||||
|
||||
function checkAllConnectionHealth() {
|
||||
var rows = document.querySelectorAll('tr[data-connection-id]');
|
||||
rows.forEach(function(row, index) {
|
||||
var connectionId = row.getAttribute('data-connection-id');
|
||||
var statusEl = row.querySelector('.connection-status');
|
||||
if (statusEl) {
|
||||
setTimeout(function() {
|
||||
checkConnectionHealth(connectionId, statusEl);
|
||||
}, index * 200);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function updateConnectionCount() {
|
||||
var countBadge = document.querySelector('.badge.bg-primary.bg-opacity-10.text-primary.fs-6');
|
||||
if (countBadge) {
|
||||
var remaining = document.querySelectorAll('tr[data-connection-id]').length;
|
||||
countBadge.textContent = remaining + ' connection' + (remaining !== 1 ? 's' : '');
|
||||
}
|
||||
}
|
||||
|
||||
function createConnectionRowHtml(conn) {
|
||||
var ak = conn.access_key || '';
|
||||
var maskedKey = ak.length > 12 ? ak.slice(0, 8) + '...' + ak.slice(-4) : ak;
|
||||
|
||||
return '<tr data-connection-id="' + window.UICore.escapeHtml(conn.id) + '">' +
|
||||
'<td class="text-center">' +
|
||||
'<span class="connection-status" data-status="checking" title="Checking...">' +
|
||||
'<span class="spinner-border spinner-border-sm text-muted" role="status" style="width: 12px; height: 12px;"></span>' +
|
||||
'</span></td>' +
|
||||
'<td><div class="d-flex align-items-center gap-2">' +
|
||||
'<div class="connection-icon"><svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" viewBox="0 0 16 16">' +
|
||||
'<path d="M4.406 3.342A5.53 5.53 0 0 1 8 2c2.69 0 4.923 2 5.166 4.579C14.758 6.804 16 8.137 16 9.773 16 11.569 14.502 13 12.687 13H3.781C1.708 13 0 11.366 0 9.318c0-1.763 1.266-3.223 2.942-3.593.143-.863.698-1.723 1.464-2.383z"/></svg></div>' +
|
||||
'<span class="fw-medium">' + window.UICore.escapeHtml(conn.name) + '</span>' +
|
||||
'</div></td>' +
|
||||
'<td><span class="text-muted small text-truncate d-inline-block" style="max-width: 200px;" title="' + window.UICore.escapeHtml(conn.endpoint_url) + '">' + window.UICore.escapeHtml(conn.endpoint_url) + '</span></td>' +
|
||||
'<td><span class="badge bg-primary bg-opacity-10 text-primary">' + window.UICore.escapeHtml(conn.region) + '</span></td>' +
|
||||
'<td><code class="small">' + window.UICore.escapeHtml(maskedKey) + '</code></td>' +
|
||||
'<td class="text-end"><div class="btn-group btn-group-sm" role="group">' +
|
||||
'<button type="button" class="btn btn-outline-secondary" data-bs-toggle="modal" data-bs-target="#editConnectionModal" ' +
|
||||
'data-id="' + window.UICore.escapeHtml(conn.id) + '" data-name="' + window.UICore.escapeHtml(conn.name) + '" ' +
|
||||
'data-endpoint="' + window.UICore.escapeHtml(conn.endpoint_url) + '" data-region="' + window.UICore.escapeHtml(conn.region) + '" ' +
|
||||
'data-access="' + window.UICore.escapeHtml(conn.access_key) + '" title="Edit connection">' +
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" viewBox="0 0 16 16">' +
|
||||
'<path d="M12.146.146a.5.5 0 0 1 .708 0l3 3a.5.5 0 0 1 0 .708l-10 10a.5.5 0 0 1-.168.11l-5 2a.5.5 0 0 1-.65-.65l2-5a.5.5 0 0 1 .11-.168l10-10zM11.207 2.5 13.5 4.793 14.793 3.5 12.5 1.207 11.207 2.5zm1.586 3L10.5 3.207 4 9.707V10h.5a.5.5 0 0 1 .5.5v.5h.5a.5.5 0 0 1 .5.5v.5h.293l6.5-6.5z"/></svg></button>' +
|
||||
'<button type="button" class="btn btn-outline-danger" data-bs-toggle="modal" data-bs-target="#deleteConnectionModal" ' +
|
||||
'data-id="' + window.UICore.escapeHtml(conn.id) + '" data-name="' + window.UICore.escapeHtml(conn.name) + '" title="Delete connection">' +
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" viewBox="0 0 16 16">' +
|
||||
'<path d="M5.5 5.5A.5.5 0 0 1 6 6v6a.5.5 0 0 1-1 0V6a.5.5 0 0 1 .5-.5zm2.5 0a.5.5 0 0 1 .5.5v6a.5.5 0 0 1-1 0V6a.5.5 0 0 1 .5-.5zm3 .5a.5.5 0 0 0-1 0v6a.5.5 0 0 0 1 0V6z"/>' +
|
||||
'<path fill-rule="evenodd" d="M14.5 3a1 1 0 0 1-1 1H13v9a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2V4h-.5a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1H6a1 1 0 0 1 1-1h2a1 1 0 0 1 1 1h3.5a1 1 0 0 1 1 1v1zM4.118 4 4 4.059V13a1 1 0 0 0 1 1h6a1 1 0 0 0 1-1V4.059L11.882 4H4.118zM2.5 3V2h11v1h-11z"/></svg></button>' +
|
||||
'</div></td></tr>';
|
||||
}
|
||||
|
||||
function setupEventListeners() {
|
||||
var testBtn = document.getElementById('testConnectionBtn');
|
||||
if (testBtn) {
|
||||
testBtn.addEventListener('click', function() {
|
||||
testConnection('createConnectionForm', 'testResult');
|
||||
});
|
||||
}
|
||||
|
||||
var editTestBtn = document.getElementById('editTestConnectionBtn');
|
||||
if (editTestBtn) {
|
||||
editTestBtn.addEventListener('click', function() {
|
||||
testConnection('editConnectionForm', 'editTestResult');
|
||||
});
|
||||
}
|
||||
|
||||
var editModal = document.getElementById('editConnectionModal');
|
||||
if (editModal) {
|
||||
editModal.addEventListener('show.bs.modal', function(event) {
|
||||
var button = event.relatedTarget;
|
||||
if (!button) return;
|
||||
|
||||
var id = button.getAttribute('data-id');
|
||||
|
||||
document.getElementById('edit_name').value = button.getAttribute('data-name') || '';
|
||||
document.getElementById('edit_endpoint_url').value = button.getAttribute('data-endpoint') || '';
|
||||
document.getElementById('edit_region').value = button.getAttribute('data-region') || '';
|
||||
document.getElementById('edit_access_key').value = button.getAttribute('data-access') || '';
|
||||
document.getElementById('edit_secret_key').value = '';
|
||||
document.getElementById('edit_secret_key').placeholder = '(unchanged — leave blank to keep current)';
|
||||
document.getElementById('edit_secret_key').required = false;
|
||||
document.getElementById('editTestResult').innerHTML = '';
|
||||
|
||||
var form = document.getElementById('editConnectionForm');
|
||||
form.action = endpoints.updateTemplate.replace('CONNECTION_ID', id);
|
||||
});
|
||||
}
|
||||
|
||||
var deleteModal = document.getElementById('deleteConnectionModal');
|
||||
if (deleteModal) {
|
||||
deleteModal.addEventListener('show.bs.modal', function(event) {
|
||||
var button = event.relatedTarget;
|
||||
if (!button) return;
|
||||
|
||||
var id = button.getAttribute('data-id');
|
||||
var name = button.getAttribute('data-name');
|
||||
|
||||
document.getElementById('deleteConnectionName').textContent = name;
|
||||
var form = document.getElementById('deleteConnectionForm');
|
||||
form.action = endpoints.deleteTemplate.replace('CONNECTION_ID', id);
|
||||
});
|
||||
}
|
||||
|
||||
var createForm = document.getElementById('createConnectionForm');
|
||||
if (createForm) {
|
||||
createForm.addEventListener('submit', function(e) {
|
||||
e.preventDefault();
|
||||
window.UICore.submitFormAjax(createForm, {
|
||||
successMessage: 'Connection created',
|
||||
onSuccess: function(data) {
|
||||
createForm.reset();
|
||||
document.getElementById('testResult').innerHTML = '';
|
||||
|
||||
if (data.connection) {
|
||||
var emptyState = document.querySelector('.empty-state');
|
||||
if (emptyState) {
|
||||
var cardBody = emptyState.closest('.card-body');
|
||||
if (cardBody) {
|
||||
cardBody.innerHTML = '<div class="table-responsive"><table class="table table-hover align-middle mb-0">' +
|
||||
'<thead class="table-light"><tr>' +
|
||||
'<th scope="col" style="width: 50px;">Status</th>' +
|
||||
'<th scope="col">Name</th><th scope="col">Endpoint</th>' +
|
||||
'<th scope="col">Region</th><th scope="col">Access Key</th>' +
|
||||
'<th scope="col" class="text-end">Actions</th></tr></thead>' +
|
||||
'<tbody></tbody></table></div>';
|
||||
}
|
||||
}
|
||||
|
||||
var tbody = document.querySelector('table tbody');
|
||||
if (tbody) {
|
||||
tbody.insertAdjacentHTML('beforeend', createConnectionRowHtml(data.connection));
|
||||
var newRow = tbody.lastElementChild;
|
||||
var statusEl = newRow.querySelector('.connection-status');
|
||||
if (statusEl) {
|
||||
checkConnectionHealth(data.connection.id, statusEl);
|
||||
}
|
||||
}
|
||||
updateConnectionCount();
|
||||
} else {
|
||||
location.reload();
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
var editForm = document.getElementById('editConnectionForm');
|
||||
if (editForm) {
|
||||
editForm.addEventListener('submit', function(e) {
|
||||
e.preventDefault();
|
||||
window.UICore.submitFormAjax(editForm, {
|
||||
successMessage: 'Connection updated',
|
||||
onSuccess: function(data) {
|
||||
var modal = bootstrap.Modal.getInstance(document.getElementById('editConnectionModal'));
|
||||
if (modal) modal.hide();
|
||||
|
||||
var connId = editForm.action.split('/').slice(-2)[0];
|
||||
var row = document.querySelector('tr[data-connection-id="' + connId + '"]');
|
||||
if (row && data.connection) {
|
||||
var nameCell = row.querySelector('.fw-medium');
|
||||
if (nameCell) nameCell.textContent = data.connection.name;
|
||||
|
||||
var endpointCell = row.querySelector('.text-truncate');
|
||||
if (endpointCell) {
|
||||
endpointCell.textContent = data.connection.endpoint_url;
|
||||
endpointCell.title = data.connection.endpoint_url;
|
||||
}
|
||||
|
||||
var regionBadge = row.querySelector('.badge.bg-primary');
|
||||
if (regionBadge) regionBadge.textContent = data.connection.region;
|
||||
|
||||
var accessCode = row.querySelector('code.small');
|
||||
if (accessCode && data.connection.access_key) {
|
||||
var ak = data.connection.access_key;
|
||||
accessCode.textContent = ak.slice(0, 8) + '...' + ak.slice(-4);
|
||||
}
|
||||
|
||||
var editBtn = row.querySelector('[data-bs-target="#editConnectionModal"]');
|
||||
if (editBtn) {
|
||||
editBtn.setAttribute('data-name', data.connection.name);
|
||||
editBtn.setAttribute('data-endpoint', data.connection.endpoint_url);
|
||||
editBtn.setAttribute('data-region', data.connection.region);
|
||||
editBtn.setAttribute('data-access', data.connection.access_key);
|
||||
}
|
||||
|
||||
var deleteBtn = row.querySelector('[data-bs-target="#deleteConnectionModal"]');
|
||||
if (deleteBtn) {
|
||||
deleteBtn.setAttribute('data-name', data.connection.name);
|
||||
}
|
||||
|
||||
var statusEl = row.querySelector('.connection-status');
|
||||
if (statusEl) {
|
||||
checkConnectionHealth(connId, statusEl);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
var deleteForm = document.getElementById('deleteConnectionForm');
|
||||
if (deleteForm) {
|
||||
deleteForm.addEventListener('submit', function(e) {
|
||||
e.preventDefault();
|
||||
window.UICore.submitFormAjax(deleteForm, {
|
||||
successMessage: 'Connection deleted',
|
||||
onSuccess: function(data) {
|
||||
var modal = bootstrap.Modal.getInstance(document.getElementById('deleteConnectionModal'));
|
||||
if (modal) modal.hide();
|
||||
|
||||
var connId = deleteForm.action.split('/').slice(-2)[0];
|
||||
var row = document.querySelector('tr[data-connection-id="' + connId + '"]');
|
||||
if (row) {
|
||||
row.remove();
|
||||
}
|
||||
|
||||
updateConnectionCount();
|
||||
|
||||
if (document.querySelectorAll('tr[data-connection-id]').length === 0) {
|
||||
location.reload();
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
init: init,
|
||||
togglePassword: togglePassword,
|
||||
testConnection: testConnection,
|
||||
checkConnectionHealth: checkConnectionHealth
|
||||
};
|
||||
})();
|
||||
664
static/js/iam-management.js
Normal file
664
static/js/iam-management.js
Normal file
@@ -0,0 +1,664 @@
|
||||
window.IAMManagement = (function() {
|
||||
'use strict';
|
||||
|
||||
var users = [];
|
||||
var currentUserKey = null;
|
||||
var endpoints = {};
|
||||
var csrfToken = '';
|
||||
var iamLocked = false;
|
||||
|
||||
var policyModal = null;
|
||||
var editUserModal = null;
|
||||
var deleteUserModal = null;
|
||||
var rotateSecretModal = null;
|
||||
var currentRotateKey = null;
|
||||
var currentEditKey = null;
|
||||
var currentDeleteKey = null;
|
||||
|
||||
var ALL_S3_ACTIONS = ['list', 'read', 'write', 'delete', 'share', 'policy', 'replication', 'lifecycle', 'cors'];
|
||||
|
||||
var policyTemplates = {
|
||||
full: [{ bucket: '*', actions: ['list', 'read', 'write', 'delete', 'share', 'policy', 'replication', 'lifecycle', 'cors', 'iam:*'] }],
|
||||
readonly: [{ bucket: '*', actions: ['list', 'read'] }],
|
||||
writer: [{ bucket: '*', actions: ['list', 'read', 'write'] }]
|
||||
};
|
||||
|
||||
function isAdminUser(policies) {
|
||||
if (!policies || !policies.length) return false;
|
||||
return policies.some(function(p) {
|
||||
return p.actions && (p.actions.indexOf('iam:*') >= 0 || p.actions.indexOf('*') >= 0);
|
||||
});
|
||||
}
|
||||
|
||||
function getPermissionLevel(actions) {
|
||||
if (!actions || !actions.length) return 'Custom (0)';
|
||||
if (actions.indexOf('*') >= 0) return 'Full Access';
|
||||
if (actions.length >= ALL_S3_ACTIONS.length) {
|
||||
var hasAll = ALL_S3_ACTIONS.every(function(a) { return actions.indexOf(a) >= 0; });
|
||||
if (hasAll) return 'Full Access';
|
||||
}
|
||||
var has = function(a) { return actions.indexOf(a) >= 0; };
|
||||
if (has('list') && has('read') && has('write') && has('delete')) return 'Read + Write + Delete';
|
||||
if (has('list') && has('read') && has('write')) return 'Read + Write';
|
||||
if (has('list') && has('read')) return 'Read Only';
|
||||
return 'Custom (' + actions.length + ')';
|
||||
}
|
||||
|
||||
function getBucketLabel(bucket) {
|
||||
return bucket === '*' ? 'All Buckets' : bucket;
|
||||
}
|
||||
|
||||
function init(config) {
|
||||
users = config.users || [];
|
||||
currentUserKey = config.currentUserKey || null;
|
||||
endpoints = config.endpoints || {};
|
||||
csrfToken = config.csrfToken || '';
|
||||
iamLocked = config.iamLocked || false;
|
||||
|
||||
if (iamLocked) return;
|
||||
|
||||
initModals();
|
||||
setupJsonAutoIndent();
|
||||
setupCopyButtons();
|
||||
setupPolicyEditor();
|
||||
setupCreateUserModal();
|
||||
setupEditUserModal();
|
||||
setupDeleteUserModal();
|
||||
setupRotateSecretModal();
|
||||
setupFormHandlers();
|
||||
setupSearch();
|
||||
setupCopyAccessKeyButtons();
|
||||
}
|
||||
|
||||
function initModals() {
|
||||
var policyModalEl = document.getElementById('policyEditorModal');
|
||||
var editModalEl = document.getElementById('editUserModal');
|
||||
var deleteModalEl = document.getElementById('deleteUserModal');
|
||||
var rotateModalEl = document.getElementById('rotateSecretModal');
|
||||
|
||||
if (policyModalEl) policyModal = new bootstrap.Modal(policyModalEl);
|
||||
if (editModalEl) editUserModal = new bootstrap.Modal(editModalEl);
|
||||
if (deleteModalEl) deleteUserModal = new bootstrap.Modal(deleteModalEl);
|
||||
if (rotateModalEl) rotateSecretModal = new bootstrap.Modal(rotateModalEl);
|
||||
}
|
||||
|
||||
function setupJsonAutoIndent() {
|
||||
window.UICore.setupJsonAutoIndent(document.getElementById('policyEditorDocument'));
|
||||
window.UICore.setupJsonAutoIndent(document.getElementById('createUserPolicies'));
|
||||
}
|
||||
|
||||
function setupCopyButtons() {
|
||||
document.querySelectorAll('.config-copy').forEach(function(button) {
|
||||
button.addEventListener('click', async function() {
|
||||
var targetId = button.dataset.copyTarget;
|
||||
var target = document.getElementById(targetId);
|
||||
if (!target) return;
|
||||
await window.UICore.copyToClipboard(target.innerText, button, 'Copy JSON');
|
||||
});
|
||||
});
|
||||
|
||||
var secretCopyButton = document.querySelector('[data-secret-copy]');
|
||||
if (secretCopyButton) {
|
||||
secretCopyButton.addEventListener('click', async function() {
|
||||
var secretInput = document.getElementById('disclosedSecretValue');
|
||||
if (!secretInput) return;
|
||||
await window.UICore.copyToClipboard(secretInput.value, secretCopyButton, 'Copy');
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function getUserPolicies(accessKey) {
|
||||
var user = users.find(function(u) { return u.access_key === accessKey; });
|
||||
return user ? JSON.stringify(user.policies, null, 2) : '';
|
||||
}
|
||||
|
||||
function applyPolicyTemplate(name, textareaEl) {
|
||||
if (policyTemplates[name] && textareaEl) {
|
||||
textareaEl.value = JSON.stringify(policyTemplates[name], null, 2);
|
||||
}
|
||||
}
|
||||
|
||||
function setupPolicyEditor() {
|
||||
var userLabelEl = document.getElementById('policyEditorUserLabel');
|
||||
var userInputEl = document.getElementById('policyEditorUser');
|
||||
var textareaEl = document.getElementById('policyEditorDocument');
|
||||
|
||||
document.querySelectorAll('[data-policy-template]').forEach(function(button) {
|
||||
button.addEventListener('click', function() {
|
||||
applyPolicyTemplate(button.dataset.policyTemplate, textareaEl);
|
||||
});
|
||||
});
|
||||
|
||||
document.querySelectorAll('[data-policy-editor]').forEach(function(button) {
|
||||
button.addEventListener('click', function() {
|
||||
var key = button.getAttribute('data-access-key');
|
||||
if (!key) return;
|
||||
|
||||
userLabelEl.textContent = key;
|
||||
userInputEl.value = key;
|
||||
textareaEl.value = getUserPolicies(key);
|
||||
|
||||
policyModal.show();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function setupCreateUserModal() {
|
||||
var createUserPoliciesEl = document.getElementById('createUserPolicies');
|
||||
|
||||
document.querySelectorAll('[data-create-policy-template]').forEach(function(button) {
|
||||
button.addEventListener('click', function() {
|
||||
applyPolicyTemplate(button.dataset.createPolicyTemplate, createUserPoliciesEl);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function setupEditUserModal() {
|
||||
var editUserForm = document.getElementById('editUserForm');
|
||||
var editUserDisplayName = document.getElementById('editUserDisplayName');
|
||||
|
||||
document.querySelectorAll('[data-edit-user]').forEach(function(btn) {
|
||||
btn.addEventListener('click', function() {
|
||||
var key = btn.dataset.editUser;
|
||||
var name = btn.dataset.displayName;
|
||||
currentEditKey = key;
|
||||
editUserDisplayName.value = name;
|
||||
editUserForm.action = endpoints.updateUser.replace('ACCESS_KEY', key);
|
||||
editUserModal.show();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function setupDeleteUserModal() {
|
||||
var deleteUserForm = document.getElementById('deleteUserForm');
|
||||
var deleteUserLabel = document.getElementById('deleteUserLabel');
|
||||
var deleteSelfWarning = document.getElementById('deleteSelfWarning');
|
||||
|
||||
document.querySelectorAll('[data-delete-user]').forEach(function(btn) {
|
||||
btn.addEventListener('click', function() {
|
||||
var key = btn.dataset.deleteUser;
|
||||
currentDeleteKey = key;
|
||||
deleteUserLabel.textContent = key;
|
||||
deleteUserForm.action = endpoints.deleteUser.replace('ACCESS_KEY', key);
|
||||
|
||||
if (key === currentUserKey) {
|
||||
deleteSelfWarning.classList.remove('d-none');
|
||||
} else {
|
||||
deleteSelfWarning.classList.add('d-none');
|
||||
}
|
||||
|
||||
deleteUserModal.show();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function setupRotateSecretModal() {
|
||||
var rotateUserLabel = document.getElementById('rotateUserLabel');
|
||||
var confirmRotateBtn = document.getElementById('confirmRotateBtn');
|
||||
var rotateCancelBtn = document.getElementById('rotateCancelBtn');
|
||||
var rotateDoneBtn = document.getElementById('rotateDoneBtn');
|
||||
var rotateSecretConfirm = document.getElementById('rotateSecretConfirm');
|
||||
var rotateSecretResult = document.getElementById('rotateSecretResult');
|
||||
var newSecretKeyInput = document.getElementById('newSecretKey');
|
||||
var copyNewSecretBtn = document.getElementById('copyNewSecret');
|
||||
|
||||
document.querySelectorAll('[data-rotate-user]').forEach(function(btn) {
|
||||
btn.addEventListener('click', function() {
|
||||
currentRotateKey = btn.dataset.rotateUser;
|
||||
rotateUserLabel.textContent = currentRotateKey;
|
||||
|
||||
rotateSecretConfirm.classList.remove('d-none');
|
||||
rotateSecretResult.classList.add('d-none');
|
||||
confirmRotateBtn.classList.remove('d-none');
|
||||
rotateCancelBtn.classList.remove('d-none');
|
||||
rotateDoneBtn.classList.add('d-none');
|
||||
|
||||
rotateSecretModal.show();
|
||||
});
|
||||
});
|
||||
|
||||
if (confirmRotateBtn) {
|
||||
confirmRotateBtn.addEventListener('click', async function() {
|
||||
if (!currentRotateKey) return;
|
||||
|
||||
window.UICore.setButtonLoading(confirmRotateBtn, true, 'Rotating...');
|
||||
|
||||
try {
|
||||
var url = endpoints.rotateSecret.replace('ACCESS_KEY', currentRotateKey);
|
||||
var response = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Accept': 'application/json',
|
||||
'X-CSRFToken': csrfToken
|
||||
}
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
var data = await response.json();
|
||||
throw new Error(data.error || 'Failed to rotate secret');
|
||||
}
|
||||
|
||||
var data = await response.json();
|
||||
newSecretKeyInput.value = data.secret_key;
|
||||
|
||||
rotateSecretConfirm.classList.add('d-none');
|
||||
rotateSecretResult.classList.remove('d-none');
|
||||
confirmRotateBtn.classList.add('d-none');
|
||||
rotateCancelBtn.classList.add('d-none');
|
||||
rotateDoneBtn.classList.remove('d-none');
|
||||
|
||||
} catch (err) {
|
||||
if (window.showToast) {
|
||||
window.showToast(err.message, 'Error', 'danger');
|
||||
}
|
||||
rotateSecretModal.hide();
|
||||
} finally {
|
||||
window.UICore.setButtonLoading(confirmRotateBtn, false);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (copyNewSecretBtn) {
|
||||
copyNewSecretBtn.addEventListener('click', async function() {
|
||||
await window.UICore.copyToClipboard(newSecretKeyInput.value, copyNewSecretBtn, 'Copy');
|
||||
});
|
||||
}
|
||||
|
||||
if (rotateDoneBtn) {
|
||||
rotateDoneBtn.addEventListener('click', function() {
|
||||
window.location.reload();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function createUserCardHtml(accessKey, displayName, policies) {
|
||||
var admin = isAdminUser(policies);
|
||||
var cardClass = 'card h-100 iam-user-card' + (admin ? ' iam-admin-card' : '');
|
||||
var roleBadge = admin
|
||||
? '<span class="iam-role-badge iam-role-admin" data-role-badge>Admin</span>'
|
||||
: '<span class="iam-role-badge iam-role-user" data-role-badge>User</span>';
|
||||
|
||||
var policyBadges = '';
|
||||
if (policies && policies.length > 0) {
|
||||
policyBadges = policies.map(function(p) {
|
||||
var bucketLabel = getBucketLabel(p.bucket);
|
||||
var permLevel = getPermissionLevel(p.actions);
|
||||
return '<span class="iam-perm-badge">' +
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" width="10" height="10" fill="currentColor" class="me-1" viewBox="0 0 16 16">' +
|
||||
'<path d="M2.522 5H2a.5.5 0 0 0-.494.574l1.372 9.149A1.5 1.5 0 0 0 4.36 16h7.278a1.5 1.5 0 0 0 1.483-1.277l1.373-9.149A.5.5 0 0 0 14 5h-.522A5.5 5.5 0 0 0 2.522 5zm1.005 0a4.5 4.5 0 0 1 8.945 0H3.527z"/>' +
|
||||
'</svg>' + window.UICore.escapeHtml(bucketLabel) + ' · ' + window.UICore.escapeHtml(permLevel) + '</span>';
|
||||
}).join('');
|
||||
} else {
|
||||
policyBadges = '<span class="badge bg-secondary bg-opacity-10 text-secondary">No policies</span>';
|
||||
}
|
||||
|
||||
var esc = window.UICore.escapeHtml;
|
||||
return '<div class="col-md-6 col-xl-4 iam-user-item" data-display-name="' + esc(displayName.toLowerCase()) + '" data-access-key-filter="' + esc(accessKey.toLowerCase()) + '">' +
|
||||
'<div class="' + cardClass + '">' +
|
||||
'<div class="card-body">' +
|
||||
'<div class="d-flex align-items-start justify-content-between mb-3">' +
|
||||
'<div class="d-flex align-items-center gap-3 min-width-0 overflow-hidden">' +
|
||||
'<div class="user-avatar user-avatar-lg flex-shrink-0">' +
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="currentColor" viewBox="0 0 16 16">' +
|
||||
'<path d="M8 8a3 3 0 1 0 0-6 3 3 0 0 0 0 6zm2-3a2 2 0 1 1-4 0 2 2 0 0 1 4 0zm4 8c0 1-1 1-1 1H3s-1 0-1-1 1-4 6-4 6 3 6 4zm-1-.004c-.001-.246-.154-.986-.832-1.664C11.516 10.68 10.289 10 8 10c-2.29 0-3.516.68-4.168 1.332-.678.678-.83 1.418-.832 1.664h10z"/>' +
|
||||
'</svg></div>' +
|
||||
'<div class="min-width-0">' +
|
||||
'<div class="d-flex align-items-center gap-2 mb-0">' +
|
||||
'<h6 class="fw-semibold mb-0 text-truncate" title="' + esc(displayName) + '">' + esc(displayName) + '</h6>' +
|
||||
roleBadge +
|
||||
'</div>' +
|
||||
'<div class="d-flex align-items-center gap-1">' +
|
||||
'<code class="small text-muted text-truncate" title="' + esc(accessKey) + '">' + esc(accessKey) + '</code>' +
|
||||
'<button type="button" class="iam-copy-key" title="Copy access key" data-copy-access-key="' + esc(accessKey) + '">' +
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" width="12" height="12" fill="currentColor" viewBox="0 0 16 16">' +
|
||||
'<path d="M4 1.5H3a2 2 0 0 0-2 2V14a2 2 0 0 0 2 2h10a2 2 0 0 0 2-2V3.5a2 2 0 0 0-2-2h-1v1h1a1 1 0 0 1 1 1V14a1 1 0 0 1-1 1H3a1 1 0 0 1-1-1V3.5a1 1 0 0 1 1-1h1v-1z"/>' +
|
||||
'<path d="M9.5 1a.5.5 0 0 1 .5.5v1a.5.5 0 0 1-.5.5h-3a.5.5 0 0 1-.5-.5v-1a.5.5 0 0 1 .5-.5h3zm-3-1A1.5 1.5 0 0 0 5 1.5v1A1.5 1.5 0 0 0 6.5 4h3A1.5 1.5 0 0 0 11 2.5v-1A1.5 1.5 0 0 0 9.5 0h-3z"/>' +
|
||||
'</svg></button>' +
|
||||
'</div>' +
|
||||
'</div></div>' +
|
||||
'<div class="dropdown flex-shrink-0">' +
|
||||
'<button class="btn btn-sm btn-icon" type="button" data-bs-toggle="dropdown" aria-expanded="false">' +
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" viewBox="0 0 16 16">' +
|
||||
'<path d="M9.5 13a1.5 1.5 0 1 1-3 0 1.5 1.5 0 0 1 3 0zm0-5a1.5 1.5 0 1 1-3 0 1.5 1.5 0 0 1 3 0zm0-5a1.5 1.5 0 1 1-3 0 1.5 1.5 0 0 1 3 0z"/>' +
|
||||
'</svg></button>' +
|
||||
'<ul class="dropdown-menu dropdown-menu-end">' +
|
||||
'<li><button class="dropdown-item" type="button" data-edit-user="' + esc(accessKey) + '" data-display-name="' + esc(displayName) + '">' +
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-2" viewBox="0 0 16 16"><path d="M12.146.146a.5.5 0 0 1 .708 0l3 3a.5.5 0 0 1 0 .708l-10 10a.5.5 0 0 1-.168.11l-5 2a.5.5 0 0 1-.65-.65l2-5a.5.5 0 0 1 .11-.168l10-10zM11.207 2.5 13.5 4.793 14.793 3.5 12.5 1.207 11.207 2.5zm1.586 3L10.5 3.207 4 9.707V10h.5a.5.5 0 0 1 .5.5v.5h.5a.5.5 0 0 1 .5.5v.5h.293l6.5-6.5z"/></svg>Edit Name</button></li>' +
|
||||
'<li><button class="dropdown-item" type="button" data-rotate-user="' + esc(accessKey) + '">' +
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-2" viewBox="0 0 16 16"><path d="M11.534 7h3.932a.25.25 0 0 1 .192.41l-1.966 2.36a.25.25 0 0 1-.384 0l-1.966-2.36a.25.25 0 0 1 .192-.41zm-11 2h3.932a.25.25 0 0 0 .192-.41L2.692 6.23a.25.25 0 0 0-.384 0L.342 8.59A.25.25 0 0 0 .534 9z"/><path fill-rule="evenodd" d="M8 3c-1.552 0-2.94.707-3.857 1.818a.5.5 0 1 1-.771-.636A6.002 6.002 0 0 1 13.917 7H12.9A5.002 5.002 0 0 0 8 3zM3.1 9a5.002 5.002 0 0 0 8.757 2.182.5.5 0 1 1 .771.636A6.002 6.002 0 0 1 2.083 9H3.1z"/></svg>Rotate Secret</button></li>' +
|
||||
'<li><hr class="dropdown-divider"></li>' +
|
||||
'<li><button class="dropdown-item text-danger" type="button" data-delete-user="' + esc(accessKey) + '">' +
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-2" viewBox="0 0 16 16"><path d="M5.5 5.5a.5.5 0 0 1 .5.5v6a.5.5 0 0 1-1 0v-6a.5.5 0 0 1 .5-.5zm2.5 0a.5.5 0 0 1 .5.5v6a.5.5 0 0 1-1 0v-6a.5.5 0 0 1 .5-.5zm3 .5v6a.5.5 0 0 1-1 0v-6a.5.5 0 0 1 1 0z"/><path fill-rule="evenodd" d="M14.5 3a1 1 0 0 1-1 1H13v9a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2V4h-.5a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1H6a1 1 0 0 1 1-1h2a1 1 0 0 1 1 1h3.5a1 1 0 0 1 1 1v1zM4.118 4 4 4.059V13a1 1 0 0 0 1 1h6a1 1 0 0 0 1-1V4.059L11.882 4H4.118zM2.5 3V2h11v1h-11z"/></svg>Delete User</button></li>' +
|
||||
'</ul></div></div>' +
|
||||
'<div class="mb-3">' +
|
||||
'<div class="small text-muted mb-2">Bucket Permissions</div>' +
|
||||
'<div class="d-flex flex-wrap gap-1" data-policy-badges>' + policyBadges + '</div></div>' +
|
||||
'<button class="btn btn-outline-primary btn-sm w-100" type="button" data-policy-editor data-access-key="' + esc(accessKey) + '">' +
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-1" viewBox="0 0 16 16"><path d="M8 4.754a3.246 3.246 0 1 0 0 6.492 3.246 3.246 0 0 0 0-6.492zM5.754 8a2.246 2.246 0 1 1 4.492 0 2.246 2.246 0 0 1-4.492 0z"/><path d="M9.796 1.343c-.527-1.79-3.065-1.79-3.592 0l-.094.319a.873.873 0 0 1-1.255.52l-.292-.16c-1.64-.892-3.433.902-2.54 2.541l.159.292a.873.873 0 0 1-.52 1.255l-.319.094c-1.79.527-1.79 3.065 0 3.592l.319.094a.873.873 0 0 1 .52 1.255l-.16.292c-.892 1.64.901 3.434 2.541 2.54l.292-.159a.873.873 0 0 1 1.255.52l.094.319c.527 1.79 3.065 1.79 3.592 0l.094-.319a.873.873 0 0 1 1.255-.52l.292.16c1.64.893 3.434-.902 2.54-2.541l-.159-.292a.873.873 0 0 1 .52-1.255l.319-.094c1.79-.527 1.79-3.065 0-3.592l-.319-.094a.873.873 0 0 1-.52-1.255l.16-.292c.893-1.64-.902-3.433-2.541-2.54l-.292.159a.873.873 0 0 1-1.255-.52l-.094-.319z"/></svg>Manage Policies</button>' +
|
||||
'</div></div></div>';
|
||||
}
|
||||
|
||||
function attachUserCardHandlers(cardElement, accessKey, displayName) {
|
||||
var editBtn = cardElement.querySelector('[data-edit-user]');
|
||||
if (editBtn) {
|
||||
editBtn.addEventListener('click', function() {
|
||||
currentEditKey = accessKey;
|
||||
document.getElementById('editUserDisplayName').value = displayName;
|
||||
document.getElementById('editUserForm').action = endpoints.updateUser.replace('ACCESS_KEY', accessKey);
|
||||
editUserModal.show();
|
||||
});
|
||||
}
|
||||
|
||||
var deleteBtn = cardElement.querySelector('[data-delete-user]');
|
||||
if (deleteBtn) {
|
||||
deleteBtn.addEventListener('click', function() {
|
||||
currentDeleteKey = accessKey;
|
||||
document.getElementById('deleteUserLabel').textContent = accessKey;
|
||||
document.getElementById('deleteUserForm').action = endpoints.deleteUser.replace('ACCESS_KEY', accessKey);
|
||||
var deleteSelfWarning = document.getElementById('deleteSelfWarning');
|
||||
if (accessKey === currentUserKey) {
|
||||
deleteSelfWarning.classList.remove('d-none');
|
||||
} else {
|
||||
deleteSelfWarning.classList.add('d-none');
|
||||
}
|
||||
deleteUserModal.show();
|
||||
});
|
||||
}
|
||||
|
||||
var rotateBtn = cardElement.querySelector('[data-rotate-user]');
|
||||
if (rotateBtn) {
|
||||
rotateBtn.addEventListener('click', function() {
|
||||
currentRotateKey = accessKey;
|
||||
document.getElementById('rotateUserLabel').textContent = accessKey;
|
||||
document.getElementById('rotateSecretConfirm').classList.remove('d-none');
|
||||
document.getElementById('rotateSecretResult').classList.add('d-none');
|
||||
document.getElementById('confirmRotateBtn').classList.remove('d-none');
|
||||
document.getElementById('rotateCancelBtn').classList.remove('d-none');
|
||||
document.getElementById('rotateDoneBtn').classList.add('d-none');
|
||||
rotateSecretModal.show();
|
||||
});
|
||||
}
|
||||
|
||||
var policyBtn = cardElement.querySelector('[data-policy-editor]');
|
||||
if (policyBtn) {
|
||||
policyBtn.addEventListener('click', function() {
|
||||
document.getElementById('policyEditorUserLabel').textContent = accessKey;
|
||||
document.getElementById('policyEditorUser').value = accessKey;
|
||||
document.getElementById('policyEditorDocument').value = getUserPolicies(accessKey);
|
||||
policyModal.show();
|
||||
});
|
||||
}
|
||||
|
||||
var copyBtn = cardElement.querySelector('[data-copy-access-key]');
|
||||
if (copyBtn) {
|
||||
copyBtn.addEventListener('click', function() {
|
||||
copyAccessKey(copyBtn);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function updateUserCount() {
|
||||
var countEl = document.querySelector('.card-header .text-muted.small');
|
||||
if (countEl) {
|
||||
var count = document.querySelectorAll('.iam-user-card').length;
|
||||
countEl.textContent = count + ' user' + (count !== 1 ? 's' : '') + ' configured';
|
||||
}
|
||||
}
|
||||
|
||||
function setupFormHandlers() {
|
||||
var createUserForm = document.querySelector('#createUserModal form');
|
||||
if (createUserForm) {
|
||||
createUserForm.addEventListener('submit', function(e) {
|
||||
e.preventDefault();
|
||||
window.UICore.submitFormAjax(createUserForm, {
|
||||
successMessage: 'User created',
|
||||
onSuccess: function(data) {
|
||||
var modal = bootstrap.Modal.getInstance(document.getElementById('createUserModal'));
|
||||
if (modal) modal.hide();
|
||||
createUserForm.reset();
|
||||
|
||||
var existingAlert = document.querySelector('.alert.alert-info.border-0.shadow-sm');
|
||||
if (existingAlert) existingAlert.remove();
|
||||
|
||||
if (data.secret_key) {
|
||||
var alertHtml = '<div class="alert alert-info border-0 shadow-sm mb-4" role="alert" id="newUserSecretAlert">' +
|
||||
'<div class="d-flex align-items-start gap-2 mb-2">' +
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" class="bi bi-key flex-shrink-0 mt-1" viewBox="0 0 16 16">' +
|
||||
'<path d="M0 8a4 4 0 0 1 7.465-2H14a.5.5 0 0 1 .354.146l1.5 1.5a.5.5 0 0 1 0 .708l-1.5 1.5a.5.5 0 0 1-.708 0L13 9.207l-.646.647a.5.5 0 0 1-.708 0L11 9.207l-.646.647a.5.5 0 0 1-.708 0L9 9.207l-.646.647A.5.5 0 0 1 8 10h-.535A4 4 0 0 1 0 8zm4-3a3 3 0 1 0 2.712 4.285A.5.5 0 0 1 7.163 9h.63l.853-.854a.5.5 0 0 1 .708 0l.646.647.646-.647a.5.5 0 0 1 .708 0l.646.647.646-.647a.5.5 0 0 1 .708 0l.646.647.793-.793-1-1h-6.63a.5.5 0 0 1-.451-.285A3 3 0 0 0 4 5z"/><path d="M4 8a1 1 0 1 1-2 0 1 1 0 0 1 2 0z"/>' +
|
||||
'</svg>' +
|
||||
'<div class="flex-grow-1">' +
|
||||
'<div class="fw-semibold">New user created: <code>' + window.UICore.escapeHtml(data.access_key) + '</code></div>' +
|
||||
'<p class="mb-2 small">This secret is only shown once. Copy it now and store it securely.</p>' +
|
||||
'</div>' +
|
||||
'<button type="button" class="btn-close" data-bs-dismiss="alert" aria-label="Close"></button>' +
|
||||
'</div>' +
|
||||
'<div class="input-group">' +
|
||||
'<span class="input-group-text"><strong>Secret key</strong></span>' +
|
||||
'<input class="form-control font-monospace" type="text" value="' + window.UICore.escapeHtml(data.secret_key) + '" readonly id="newUserSecret" />' +
|
||||
'<button class="btn btn-outline-primary" type="button" id="copyNewUserSecret">Copy</button>' +
|
||||
'</div></div>';
|
||||
var container = document.querySelector('.page-header');
|
||||
if (container) {
|
||||
container.insertAdjacentHTML('afterend', alertHtml);
|
||||
document.getElementById('copyNewUserSecret').addEventListener('click', async function() {
|
||||
await window.UICore.copyToClipboard(data.secret_key, this, 'Copy');
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
var usersGrid = document.querySelector('.row.g-3');
|
||||
var emptyState = document.querySelector('.empty-state');
|
||||
if (emptyState) {
|
||||
var emptyCol = emptyState.closest('.col-12');
|
||||
if (emptyCol) emptyCol.remove();
|
||||
if (!usersGrid) {
|
||||
var cardBody = document.querySelector('.card-body.px-4.pb-4');
|
||||
if (cardBody) {
|
||||
cardBody.innerHTML = '<div class="row g-3"></div>';
|
||||
usersGrid = cardBody.querySelector('.row.g-3');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (usersGrid) {
|
||||
var cardHtml = createUserCardHtml(data.access_key, data.display_name, data.policies);
|
||||
usersGrid.insertAdjacentHTML('beforeend', cardHtml);
|
||||
var newCard = usersGrid.lastElementChild;
|
||||
attachUserCardHandlers(newCard, data.access_key, data.display_name);
|
||||
users.push({
|
||||
access_key: data.access_key,
|
||||
display_name: data.display_name,
|
||||
policies: data.policies || []
|
||||
});
|
||||
updateUserCount();
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
var policyEditorForm = document.getElementById('policyEditorForm');
|
||||
if (policyEditorForm) {
|
||||
policyEditorForm.addEventListener('submit', function(e) {
|
||||
e.preventDefault();
|
||||
var userInputEl = document.getElementById('policyEditorUser');
|
||||
var key = userInputEl.value;
|
||||
if (!key) return;
|
||||
|
||||
var template = policyEditorForm.dataset.actionTemplate;
|
||||
policyEditorForm.action = template.replace('ACCESS_KEY_PLACEHOLDER', key);
|
||||
|
||||
window.UICore.submitFormAjax(policyEditorForm, {
|
||||
successMessage: 'Policies updated',
|
||||
onSuccess: function(data) {
|
||||
policyModal.hide();
|
||||
|
||||
var userCard = document.querySelector('[data-access-key="' + key + '"]');
|
||||
if (userCard) {
|
||||
var cardEl = userCard.closest('.iam-user-card');
|
||||
var badgeContainer = cardEl ? cardEl.querySelector('[data-policy-badges]') : null;
|
||||
if (badgeContainer && data.policies) {
|
||||
var badges = data.policies.map(function(p) {
|
||||
var bl = getBucketLabel(p.bucket);
|
||||
var pl = getPermissionLevel(p.actions);
|
||||
return '<span class="iam-perm-badge">' +
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" width="10" height="10" fill="currentColor" class="me-1" viewBox="0 0 16 16">' +
|
||||
'<path d="M2.522 5H2a.5.5 0 0 0-.494.574l1.372 9.149A1.5 1.5 0 0 0 4.36 16h7.278a1.5 1.5 0 0 0 1.483-1.277l1.373-9.149A.5.5 0 0 0 14 5h-.522A5.5 5.5 0 0 0 2.522 5zm1.005 0a4.5 4.5 0 0 1 8.945 0H3.527z"/>' +
|
||||
'</svg>' + window.UICore.escapeHtml(bl) + ' · ' + window.UICore.escapeHtml(pl) + '</span>';
|
||||
}).join('');
|
||||
badgeContainer.innerHTML = badges || '<span class="badge bg-secondary bg-opacity-10 text-secondary">No policies</span>';
|
||||
}
|
||||
if (cardEl) {
|
||||
var nowAdmin = isAdminUser(data.policies);
|
||||
cardEl.classList.toggle('iam-admin-card', nowAdmin);
|
||||
var roleBadgeEl = cardEl.querySelector('[data-role-badge]');
|
||||
if (roleBadgeEl) {
|
||||
if (nowAdmin) {
|
||||
roleBadgeEl.className = 'iam-role-badge iam-role-admin';
|
||||
roleBadgeEl.textContent = 'Admin';
|
||||
} else {
|
||||
roleBadgeEl.className = 'iam-role-badge iam-role-user';
|
||||
roleBadgeEl.textContent = 'User';
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var userIndex = users.findIndex(function(u) { return u.access_key === key; });
|
||||
if (userIndex >= 0 && data.policies) {
|
||||
users[userIndex].policies = data.policies;
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
var editUserForm = document.getElementById('editUserForm');
|
||||
if (editUserForm) {
|
||||
editUserForm.addEventListener('submit', function(e) {
|
||||
e.preventDefault();
|
||||
var key = currentEditKey;
|
||||
window.UICore.submitFormAjax(editUserForm, {
|
||||
successMessage: 'User updated',
|
||||
onSuccess: function(data) {
|
||||
editUserModal.hide();
|
||||
|
||||
var newName = data.display_name || document.getElementById('editUserDisplayName').value;
|
||||
var editBtn = document.querySelector('[data-edit-user="' + key + '"]');
|
||||
if (editBtn) {
|
||||
editBtn.setAttribute('data-display-name', newName);
|
||||
var card = editBtn.closest('.iam-user-card');
|
||||
if (card) {
|
||||
var nameEl = card.querySelector('h6');
|
||||
if (nameEl) {
|
||||
nameEl.textContent = newName;
|
||||
nameEl.title = newName;
|
||||
}
|
||||
var itemWrapper = card.closest('.iam-user-item');
|
||||
if (itemWrapper) {
|
||||
itemWrapper.setAttribute('data-display-name', newName.toLowerCase());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var userIndex = users.findIndex(function(u) { return u.access_key === key; });
|
||||
if (userIndex >= 0) {
|
||||
users[userIndex].display_name = newName;
|
||||
}
|
||||
|
||||
if (key === currentUserKey) {
|
||||
document.querySelectorAll('.sidebar-user .user-name').forEach(function(el) {
|
||||
var truncated = newName.length > 16 ? newName.substring(0, 16) + '...' : newName;
|
||||
el.textContent = truncated;
|
||||
el.title = newName;
|
||||
});
|
||||
document.querySelectorAll('.sidebar-user[data-username]').forEach(function(el) {
|
||||
el.setAttribute('data-username', newName);
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
var deleteUserForm = document.getElementById('deleteUserForm');
|
||||
if (deleteUserForm) {
|
||||
deleteUserForm.addEventListener('submit', function(e) {
|
||||
e.preventDefault();
|
||||
var key = currentDeleteKey;
|
||||
window.UICore.submitFormAjax(deleteUserForm, {
|
||||
successMessage: 'User deleted',
|
||||
onSuccess: function(data) {
|
||||
deleteUserModal.hide();
|
||||
|
||||
if (key === currentUserKey) {
|
||||
window.location.href = '/ui/';
|
||||
return;
|
||||
}
|
||||
|
||||
var deleteBtn = document.querySelector('[data-delete-user="' + key + '"]');
|
||||
if (deleteBtn) {
|
||||
var cardCol = deleteBtn.closest('[class*="col-"]');
|
||||
if (cardCol) {
|
||||
cardCol.remove();
|
||||
}
|
||||
}
|
||||
|
||||
users = users.filter(function(u) { return u.access_key !== key; });
|
||||
updateUserCount();
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function setupSearch() {
|
||||
var searchInput = document.getElementById('iam-user-search');
|
||||
if (!searchInput) return;
|
||||
|
||||
searchInput.addEventListener('input', function() {
|
||||
var query = searchInput.value.toLowerCase().trim();
|
||||
var items = document.querySelectorAll('.iam-user-item');
|
||||
var noResults = document.getElementById('iam-no-results');
|
||||
var visibleCount = 0;
|
||||
|
||||
items.forEach(function(item) {
|
||||
var name = item.getAttribute('data-display-name') || '';
|
||||
var key = item.getAttribute('data-access-key-filter') || '';
|
||||
var matches = !query || name.indexOf(query) >= 0 || key.indexOf(query) >= 0;
|
||||
item.classList.toggle('d-none', !matches);
|
||||
if (matches) visibleCount++;
|
||||
});
|
||||
|
||||
if (noResults) {
|
||||
noResults.classList.toggle('d-none', visibleCount > 0);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function copyAccessKey(btn) {
|
||||
var key = btn.getAttribute('data-copy-access-key');
|
||||
if (!key) return;
|
||||
var originalHtml = btn.innerHTML;
|
||||
navigator.clipboard.writeText(key).then(function() {
|
||||
btn.innerHTML = '<svg xmlns="http://www.w3.org/2000/svg" width="12" height="12" fill="currentColor" viewBox="0 0 16 16"><path d="M13.854 3.646a.5.5 0 0 1 0 .708l-7 7a.5.5 0 0 1-.708 0l-3.5-3.5a.5.5 0 1 1 .708-.708L6.5 10.293l6.646-6.647a.5.5 0 0 1 .708 0z"/></svg>';
|
||||
btn.style.color = '#22c55e';
|
||||
setTimeout(function() {
|
||||
btn.innerHTML = originalHtml;
|
||||
btn.style.color = '';
|
||||
}, 1200);
|
||||
}).catch(function() {});
|
||||
}
|
||||
|
||||
function setupCopyAccessKeyButtons() {
|
||||
document.querySelectorAll('[data-copy-access-key]').forEach(function(btn) {
|
||||
btn.addEventListener('click', function() {
|
||||
copyAccessKey(btn);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
init: init
|
||||
};
|
||||
})();
|
||||
328
static/js/ui-core.js
Normal file
328
static/js/ui-core.js
Normal file
@@ -0,0 +1,328 @@
|
||||
window.UICore = (function() {
|
||||
'use strict';
|
||||
|
||||
function getCsrfToken() {
|
||||
const meta = document.querySelector('meta[name="csrf-token"]');
|
||||
return meta ? meta.getAttribute('content') : '';
|
||||
}
|
||||
|
||||
function formatBytes(bytes) {
|
||||
if (!Number.isFinite(bytes)) return bytes + ' bytes';
|
||||
const units = ['bytes', 'KB', 'MB', 'GB', 'TB'];
|
||||
let i = 0;
|
||||
let size = bytes;
|
||||
while (size >= 1024 && i < units.length - 1) {
|
||||
size /= 1024;
|
||||
i++;
|
||||
}
|
||||
return size.toFixed(i === 0 ? 0 : 1) + ' ' + units[i];
|
||||
}
|
||||
|
||||
function escapeHtml(value) {
|
||||
if (value === null || value === undefined) return '';
|
||||
return String(value)
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, ''');
|
||||
}
|
||||
|
||||
async function submitFormAjax(form, options) {
|
||||
options = options || {};
|
||||
var onSuccess = options.onSuccess || function() {};
|
||||
var onError = options.onError || function() {};
|
||||
var successMessage = options.successMessage || 'Operation completed';
|
||||
|
||||
var formData = new FormData(form);
|
||||
var csrfToken = getCsrfToken();
|
||||
var submitBtn = form.querySelector('[type="submit"]');
|
||||
var originalHtml = submitBtn ? submitBtn.innerHTML : '';
|
||||
|
||||
try {
|
||||
if (submitBtn) {
|
||||
submitBtn.disabled = true;
|
||||
submitBtn.innerHTML = '<span class="spinner-border spinner-border-sm me-1"></span>Saving...';
|
||||
}
|
||||
|
||||
var formAction = form.getAttribute('action') || form.action;
|
||||
var response = await fetch(formAction, {
|
||||
method: form.getAttribute('method') || 'POST',
|
||||
headers: {
|
||||
'X-CSRFToken': csrfToken,
|
||||
'Accept': 'application/json',
|
||||
'X-Requested-With': 'XMLHttpRequest'
|
||||
},
|
||||
body: formData,
|
||||
redirect: 'follow'
|
||||
});
|
||||
|
||||
var contentType = response.headers.get('content-type') || '';
|
||||
if (!contentType.includes('application/json')) {
|
||||
throw new Error('Server returned an unexpected response. Please try again.');
|
||||
}
|
||||
|
||||
var data = await response.json();
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(data.error || 'HTTP ' + response.status);
|
||||
}
|
||||
|
||||
window.showToast(data.message || successMessage, 'Success', 'success');
|
||||
onSuccess(data);
|
||||
|
||||
} catch (err) {
|
||||
window.showToast(err.message, 'Error', 'error');
|
||||
onError(err);
|
||||
} finally {
|
||||
if (submitBtn) {
|
||||
submitBtn.disabled = false;
|
||||
submitBtn.innerHTML = originalHtml;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function PollingManager() {
|
||||
this.intervals = {};
|
||||
this.callbacks = {};
|
||||
this.timers = {};
|
||||
this.defaults = {
|
||||
replication: 30000,
|
||||
lifecycle: 60000,
|
||||
connectionHealth: 60000,
|
||||
bucketStats: 120000
|
||||
};
|
||||
this._loadSettings();
|
||||
}
|
||||
|
||||
PollingManager.prototype._loadSettings = function() {
|
||||
try {
|
||||
var stored = localStorage.getItem('myfsio-polling-intervals');
|
||||
if (stored) {
|
||||
var settings = JSON.parse(stored);
|
||||
for (var key in settings) {
|
||||
if (settings.hasOwnProperty(key)) {
|
||||
this.defaults[key] = settings[key];
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn('Failed to load polling settings:', e);
|
||||
}
|
||||
};
|
||||
|
||||
PollingManager.prototype.saveSettings = function(settings) {
|
||||
try {
|
||||
for (var key in settings) {
|
||||
if (settings.hasOwnProperty(key)) {
|
||||
this.defaults[key] = settings[key];
|
||||
}
|
||||
}
|
||||
localStorage.setItem('myfsio-polling-intervals', JSON.stringify(this.defaults));
|
||||
} catch (e) {
|
||||
console.warn('Failed to save polling settings:', e);
|
||||
}
|
||||
};
|
||||
|
||||
PollingManager.prototype.start = function(key, callback, interval) {
|
||||
this.stop(key);
|
||||
var ms = interval !== undefined ? interval : (this.defaults[key] || 30000);
|
||||
if (ms <= 0) return;
|
||||
|
||||
this.callbacks[key] = callback;
|
||||
this.intervals[key] = ms;
|
||||
|
||||
callback();
|
||||
|
||||
var self = this;
|
||||
this.timers[key] = setInterval(function() {
|
||||
if (!document.hidden) {
|
||||
callback();
|
||||
}
|
||||
}, ms);
|
||||
};
|
||||
|
||||
PollingManager.prototype.stop = function(key) {
|
||||
if (this.timers[key]) {
|
||||
clearInterval(this.timers[key]);
|
||||
delete this.timers[key];
|
||||
}
|
||||
};
|
||||
|
||||
PollingManager.prototype.stopAll = function() {
|
||||
for (var key in this.timers) {
|
||||
if (this.timers.hasOwnProperty(key)) {
|
||||
clearInterval(this.timers[key]);
|
||||
}
|
||||
}
|
||||
this.timers = {};
|
||||
};
|
||||
|
||||
PollingManager.prototype.updateInterval = function(key, newInterval) {
|
||||
var callback = this.callbacks[key];
|
||||
this.defaults[key] = newInterval;
|
||||
this.saveSettings(this.defaults);
|
||||
if (callback) {
|
||||
this.start(key, callback, newInterval);
|
||||
}
|
||||
};
|
||||
|
||||
PollingManager.prototype.getSettings = function() {
|
||||
var result = {};
|
||||
for (var key in this.defaults) {
|
||||
if (this.defaults.hasOwnProperty(key)) {
|
||||
result[key] = this.defaults[key];
|
||||
}
|
||||
}
|
||||
return result;
|
||||
};
|
||||
|
||||
var pollingManager = new PollingManager();
|
||||
|
||||
document.addEventListener('visibilitychange', function() {
|
||||
if (document.hidden) {
|
||||
pollingManager.stopAll();
|
||||
} else {
|
||||
for (var key in pollingManager.callbacks) {
|
||||
if (pollingManager.callbacks.hasOwnProperty(key)) {
|
||||
pollingManager.start(key, pollingManager.callbacks[key], pollingManager.intervals[key]);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
window.addEventListener('beforeunload', function() {
|
||||
pollingManager.stopAll();
|
||||
});
|
||||
|
||||
return {
|
||||
getCsrfToken: getCsrfToken,
|
||||
formatBytes: formatBytes,
|
||||
escapeHtml: escapeHtml,
|
||||
submitFormAjax: submitFormAjax,
|
||||
PollingManager: PollingManager,
|
||||
pollingManager: pollingManager
|
||||
};
|
||||
})();
|
||||
|
||||
window.pollingManager = window.UICore.pollingManager;
|
||||
|
||||
window.UICore.copyToClipboard = async function(text, button, originalText) {
|
||||
try {
|
||||
await navigator.clipboard.writeText(text);
|
||||
if (button) {
|
||||
var prevText = button.textContent;
|
||||
button.textContent = 'Copied!';
|
||||
setTimeout(function() {
|
||||
button.textContent = originalText || prevText;
|
||||
}, 1500);
|
||||
}
|
||||
return true;
|
||||
} catch (err) {
|
||||
console.error('Copy failed:', err);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
window.UICore.setButtonLoading = function(button, isLoading, loadingText) {
|
||||
if (!button) return;
|
||||
if (isLoading) {
|
||||
button._originalHtml = button.innerHTML;
|
||||
button._originalDisabled = button.disabled;
|
||||
button.disabled = true;
|
||||
button.innerHTML = '<span class="spinner-border spinner-border-sm me-1"></span>' + (loadingText || 'Loading...');
|
||||
} else {
|
||||
button.disabled = button._originalDisabled || false;
|
||||
button.innerHTML = button._originalHtml || button.innerHTML;
|
||||
}
|
||||
};
|
||||
|
||||
window.UICore.updateBadgeCount = function(selector, count, singular, plural) {
|
||||
var badge = document.querySelector(selector);
|
||||
if (badge) {
|
||||
var label = count === 1 ? (singular || '') : (plural || 's');
|
||||
badge.textContent = count + ' ' + label;
|
||||
}
|
||||
};
|
||||
|
||||
window.UICore.setupJsonAutoIndent = function(textarea) {
|
||||
if (!textarea) return;
|
||||
|
||||
textarea.addEventListener('keydown', function(e) {
|
||||
if (e.key === 'Enter') {
|
||||
e.preventDefault();
|
||||
|
||||
var start = this.selectionStart;
|
||||
var end = this.selectionEnd;
|
||||
var value = this.value;
|
||||
|
||||
var lineStart = value.lastIndexOf('\n', start - 1) + 1;
|
||||
var currentLine = value.substring(lineStart, start);
|
||||
|
||||
var indentMatch = currentLine.match(/^(\s*)/);
|
||||
var indent = indentMatch ? indentMatch[1] : '';
|
||||
|
||||
var trimmedLine = currentLine.trim();
|
||||
var lastChar = trimmedLine.slice(-1);
|
||||
|
||||
var newIndent = indent;
|
||||
var insertAfter = '';
|
||||
|
||||
if (lastChar === '{' || lastChar === '[') {
|
||||
newIndent = indent + ' ';
|
||||
|
||||
var charAfterCursor = value.substring(start, start + 1).trim();
|
||||
if ((lastChar === '{' && charAfterCursor === '}') ||
|
||||
(lastChar === '[' && charAfterCursor === ']')) {
|
||||
insertAfter = '\n' + indent;
|
||||
}
|
||||
} else if (lastChar === ',' || lastChar === ':') {
|
||||
newIndent = indent;
|
||||
}
|
||||
|
||||
var insertion = '\n' + newIndent + insertAfter;
|
||||
var newValue = value.substring(0, start) + insertion + value.substring(end);
|
||||
|
||||
this.value = newValue;
|
||||
|
||||
var newCursorPos = start + 1 + newIndent.length;
|
||||
this.selectionStart = this.selectionEnd = newCursorPos;
|
||||
|
||||
this.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
}
|
||||
|
||||
if (e.key === 'Tab') {
|
||||
e.preventDefault();
|
||||
var start = this.selectionStart;
|
||||
var end = this.selectionEnd;
|
||||
|
||||
if (e.shiftKey) {
|
||||
var lineStart = this.value.lastIndexOf('\n', start - 1) + 1;
|
||||
var lineContent = this.value.substring(lineStart, start);
|
||||
if (lineContent.startsWith(' ')) {
|
||||
this.value = this.value.substring(0, lineStart) +
|
||||
this.value.substring(lineStart + 2);
|
||||
this.selectionStart = this.selectionEnd = Math.max(lineStart, start - 2);
|
||||
}
|
||||
} else {
|
||||
this.value = this.value.substring(0, start) + ' ' + this.value.substring(end);
|
||||
this.selectionStart = this.selectionEnd = start + 2;
|
||||
}
|
||||
|
||||
this.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
document.addEventListener('DOMContentLoaded', function() {
|
||||
var flashMessage = sessionStorage.getItem('flashMessage');
|
||||
if (flashMessage) {
|
||||
sessionStorage.removeItem('flashMessage');
|
||||
try {
|
||||
var msg = JSON.parse(flashMessage);
|
||||
if (window.showToast) {
|
||||
window.showToast(msg.body || msg.title, msg.title, msg.variant || 'info');
|
||||
}
|
||||
} catch (e) {}
|
||||
}
|
||||
});
|
||||
@@ -94,6 +94,21 @@
|
||||
</svg>
|
||||
<span>Metrics</span>
|
||||
</a>
|
||||
<a href="{{ url_for('ui.sites_dashboard') }}" class="sidebar-link {% if request.endpoint == 'ui.sites_dashboard' %}active{% endif %}">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" viewBox="0 0 16 16">
|
||||
<path d="M0 8a8 8 0 1 1 16 0A8 8 0 0 1 0 8zm7.5-6.923c-.67.204-1.335.82-1.887 1.855A7.97 7.97 0 0 0 5.145 4H7.5V1.077zM4.09 4a9.267 9.267 0 0 1 .64-1.539 6.7 6.7 0 0 1 .597-.933A7.025 7.025 0 0 0 2.255 4H4.09zm-.582 3.5c.03-.877.138-1.718.312-2.5H1.674a6.958 6.958 0 0 0-.656 2.5h2.49zM4.847 5a12.5 12.5 0 0 0-.338 2.5H7.5V5H4.847zM8.5 5v2.5h2.99a12.495 12.495 0 0 0-.337-2.5H8.5zM4.51 8.5a12.5 12.5 0 0 0 .337 2.5H7.5V8.5H4.51zm3.99 0V11h2.653c.187-.765.306-1.608.338-2.5H8.5zM5.145 12c.138.386.295.744.468 1.068.552 1.035 1.218 1.65 1.887 1.855V12H5.145zm.182 2.472a6.696 6.696 0 0 1-.597-.933A9.268 9.268 0 0 1 4.09 12H2.255a7.024 7.024 0 0 0 3.072 2.472zM3.82 11a13.652 13.652 0 0 1-.312-2.5h-2.49c.062.89.291 1.733.656 2.5H3.82zm6.853 3.472A7.024 7.024 0 0 0 13.745 12H11.91a9.27 9.27 0 0 1-.64 1.539 6.688 6.688 0 0 1-.597.933zM8.5 12v2.923c.67-.204 1.335-.82 1.887-1.855.173-.324.33-.682.468-1.068H8.5zm3.68-1h2.146c.365-.767.594-1.61.656-2.5h-2.49a13.65 13.65 0 0 1-.312 2.5zm2.802-3.5a6.959 6.959 0 0 0-.656-2.5H12.18c.174.782.282 1.623.312 2.5h2.49zM11.27 2.461c.247.464.462.98.64 1.539h1.835a7.024 7.024 0 0 0-3.072-2.472c.218.284.418.598.597.933zM10.855 4a7.966 7.966 0 0 0-.468-1.068C9.835 1.897 9.17 1.282 8.5 1.077V4h2.355z"/>
|
||||
</svg>
|
||||
<span>Sites</span>
|
||||
</a>
|
||||
{% endif %}
|
||||
{% if website_hosting_nav %}
|
||||
<a href="{{ url_for('ui.website_domains_dashboard') }}" class="sidebar-link {% if request.endpoint == 'ui.website_domains_dashboard' %}active{% endif %}">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" viewBox="0 0 16 16">
|
||||
<path d="M4.715 6.542 3.343 7.914a3 3 0 1 0 4.243 4.243l1.828-1.829A3 3 0 0 0 8.586 5.5L8 6.086a1.002 1.002 0 0 0-.154.199 2 2 0 0 1 .861 3.337L6.88 11.45a2 2 0 1 1-2.83-2.83l.793-.792a4.018 4.018 0 0 1-.128-1.287z"/>
|
||||
<path d="M6.586 4.672A3 3 0 0 0 7.414 9.5l.775-.776a2 2 0 0 1-.896-3.346L9.12 3.55a2 2 0 1 1 2.83 2.83l-.793.792c.112.42.155.855.128 1.287l1.372-1.372a3 3 0 1 0-4.243-4.243L6.586 4.672z"/>
|
||||
</svg>
|
||||
<span>Domains</span>
|
||||
</a>
|
||||
{% endif %}
|
||||
</div>
|
||||
<div class="nav-section">
|
||||
@@ -179,6 +194,21 @@
|
||||
</svg>
|
||||
<span class="sidebar-link-text">Metrics</span>
|
||||
</a>
|
||||
<a href="{{ url_for('ui.sites_dashboard') }}" class="sidebar-link {% if request.endpoint == 'ui.sites_dashboard' %}active{% endif %}" data-tooltip="Sites">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" viewBox="0 0 16 16">
|
||||
<path d="M0 8a8 8 0 1 1 16 0A8 8 0 0 1 0 8zm7.5-6.923c-.67.204-1.335.82-1.887 1.855A7.97 7.97 0 0 0 5.145 4H7.5V1.077zM4.09 4a9.267 9.267 0 0 1 .64-1.539 6.7 6.7 0 0 1 .597-.933A7.025 7.025 0 0 0 2.255 4H4.09zm-.582 3.5c.03-.877.138-1.718.312-2.5H1.674a6.958 6.958 0 0 0-.656 2.5h2.49zM4.847 5a12.5 12.5 0 0 0-.338 2.5H7.5V5H4.847zM8.5 5v2.5h2.99a12.495 12.495 0 0 0-.337-2.5H8.5zM4.51 8.5a12.5 12.5 0 0 0 .337 2.5H7.5V8.5H4.51zm3.99 0V11h2.653c.187-.765.306-1.608.338-2.5H8.5zM5.145 12c.138.386.295.744.468 1.068.552 1.035 1.218 1.65 1.887 1.855V12H5.145zm.182 2.472a6.696 6.696 0 0 1-.597-.933A9.268 9.268 0 0 1 4.09 12H2.255a7.024 7.024 0 0 0 3.072 2.472zM3.82 11a13.652 13.652 0 0 1-.312-2.5h-2.49c.062.89.291 1.733.656 2.5H3.82zm6.853 3.472A7.024 7.024 0 0 0 13.745 12H11.91a9.27 9.27 0 0 1-.64 1.539 6.688 6.688 0 0 1-.597.933zM8.5 12v2.923c.67-.204 1.335-.82 1.887-1.855.173-.324.33-.682.468-1.068H8.5zm3.68-1h2.146c.365-.767.594-1.61.656-2.5h-2.49a13.65 13.65 0 0 1-.312 2.5zm2.802-3.5a6.959 6.959 0 0 0-.656-2.5H12.18c.174.782.282 1.623.312 2.5h2.49zM11.27 2.461c.247.464.462.98.64 1.539h1.835a7.024 7.024 0 0 0-3.072-2.472c.218.284.418.598.597.933zM10.855 4a7.966 7.966 0 0 0-.468-1.068C9.835 1.897 9.17 1.282 8.5 1.077V4h2.355z"/>
|
||||
</svg>
|
||||
<span class="sidebar-link-text">Sites</span>
|
||||
</a>
|
||||
{% endif %}
|
||||
{% if website_hosting_nav %}
|
||||
<a href="{{ url_for('ui.website_domains_dashboard') }}" class="sidebar-link {% if request.endpoint == 'ui.website_domains_dashboard' %}active{% endif %}" data-tooltip="Domains">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" viewBox="0 0 16 16">
|
||||
<path d="M4.715 6.542 3.343 7.914a3 3 0 1 0 4.243 4.243l1.828-1.829A3 3 0 0 0 8.586 5.5L8 6.086a1.002 1.002 0 0 0-.154.199 2 2 0 0 1 .861 3.337L6.88 11.45a2 2 0 1 1-2.83-2.83l.793-.792a4.018 4.018 0 0 1-.128-1.287z"/>
|
||||
<path d="M6.586 4.672A3 3 0 0 0 7.414 9.5l.775-.776a2 2 0 0 1-.896-3.346L9.12 3.55a2 2 0 1 1 2.83 2.83l-.793.792c.112.42.155.855.128 1.287l1.372-1.372a3 3 0 1 0-4.243-4.243L6.586 4.672z"/>
|
||||
</svg>
|
||||
<span class="sidebar-link-text">Domains</span>
|
||||
</a>
|
||||
{% endif %}
|
||||
</div>
|
||||
<div class="nav-section">
|
||||
@@ -393,6 +423,8 @@
|
||||
{% endwith %}
|
||||
})();
|
||||
</script>
|
||||
<script src="{{ url_for('static', filename='js/ui-core.js') }}"></script>
|
||||
{% block extra_scripts %}{% endblock %}
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -51,7 +51,7 @@
|
||||
</div>
|
||||
<div>
|
||||
<h5 class="bucket-name text-break">{{ bucket.meta.name }}</h5>
|
||||
<small class="text-muted">Created {{ bucket.meta.created_at.strftime('%b %d, %Y') }}</small>
|
||||
<small class="text-muted">Created {{ bucket.meta.created_at | format_datetime }}</small>
|
||||
</div>
|
||||
</div>
|
||||
<span class="badge {{ bucket.access_badge }} bucket-access-badge">{{ bucket.access_label }}</span>
|
||||
@@ -89,6 +89,14 @@
|
||||
</div>
|
||||
</div>
|
||||
{% endfor %}
|
||||
<div class="col-12 d-none" id="bucket-no-results">
|
||||
<div class="text-center py-5 text-muted">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="32" height="32" fill="currentColor" class="mb-3 opacity-50" viewBox="0 0 16 16">
|
||||
<path d="M11.742 10.344a6.5 6.5 0 1 0-1.397 1.398h-.001c.03.04.062.078.098.115l3.85 3.85a1 1 0 0 0 1.415-1.414l-3.85-3.85a1.007 1.007 0 0 0-.115-.1zM12 6.5a5.5 5.5 0 1 1-11 0 5.5 5.5 0 0 1 11 0z"/>
|
||||
</svg>
|
||||
<p class="mb-0 fw-medium">No buckets match your filter.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="modal fade" id="createBucketModal" tabindex="-1" aria-hidden="true">
|
||||
@@ -104,7 +112,7 @@
|
||||
</h1>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
||||
</div>
|
||||
<form method="post" action="{{ url_for('ui.create_bucket') }}">
|
||||
<form method="post" action="{{ url_for('ui.create_bucket') }}" id="createBucketForm">
|
||||
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}" />
|
||||
<div class="modal-body pt-0">
|
||||
<label class="form-label fw-medium">Bucket name</label>
|
||||
@@ -141,7 +149,7 @@
|
||||
let visibleCount = 0;
|
||||
|
||||
bucketItems.forEach(item => {
|
||||
const name = item.querySelector('.card-title').textContent.toLowerCase();
|
||||
const name = item.querySelector('.bucket-name').textContent.toLowerCase();
|
||||
if (name.includes(term)) {
|
||||
item.classList.remove('d-none');
|
||||
visibleCount++;
|
||||
@@ -149,6 +157,15 @@
|
||||
item.classList.add('d-none');
|
||||
}
|
||||
});
|
||||
|
||||
var noResults = document.getElementById('bucket-no-results');
|
||||
if (noResults) {
|
||||
if (term && visibleCount === 0) {
|
||||
noResults.classList.remove('d-none');
|
||||
} else {
|
||||
noResults.classList.add('d-none');
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@@ -205,6 +222,25 @@
|
||||
});
|
||||
row.style.cursor = 'pointer';
|
||||
});
|
||||
|
||||
var createForm = document.getElementById('createBucketForm');
|
||||
if (createForm) {
|
||||
createForm.addEventListener('submit', function(e) {
|
||||
e.preventDefault();
|
||||
window.UICore.submitFormAjax(createForm, {
|
||||
successMessage: 'Bucket created',
|
||||
onSuccess: function(data) {
|
||||
var modal = bootstrap.Modal.getInstance(document.getElementById('createBucketModal'));
|
||||
if (modal) modal.hide();
|
||||
if (data.bucket_name) {
|
||||
window.location.href = '{{ url_for("ui.bucket_detail", bucket_name="__BUCKET__") }}'.replace('__BUCKET__', data.bucket_name);
|
||||
} else {
|
||||
location.reload();
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
})();
|
||||
</script>
|
||||
{% endblock %}
|
||||
|
||||
@@ -57,7 +57,7 @@
|
||||
<label for="secret_key" class="form-label fw-medium">Secret Key</label>
|
||||
<div class="input-group">
|
||||
<input type="password" class="form-control font-monospace" id="secret_key" name="secret_key" required>
|
||||
<button class="btn btn-outline-secondary" type="button" onclick="togglePassword('secret_key')" title="Toggle visibility">
|
||||
<button class="btn btn-outline-secondary" type="button" onclick="ConnectionsManagement.togglePassword('secret_key')" title="Toggle visibility">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" viewBox="0 0 16 16">
|
||||
<path d="M16 8s-3-5.5-8-5.5S0 8 0 8s3 5.5 8 5.5S16 8 16 8zM1.173 8a13.133 13.133 0 0 1 1.66-2.043C4.12 4.668 5.88 3.5 8 3.5c2.12 0 3.879 1.168 5.168 2.457A13.133 13.133 0 0 1 14.828 8c-.058.087-.122.183-.195.288-.335.48-.83 1.12-1.465 1.755C11.879 11.332 10.119 12.5 8 12.5c-2.12 0-3.879-1.168-5.168-2.457A13.134 13.134 0 0 1 1.172 8z"/>
|
||||
<path d="M8 5.5a2.5 2.5 0 1 0 0 5 2.5 2.5 0 0 0 0-5zM4.5 8a3.5 3.5 0 1 1 7 0 3.5 3.5 0 0 1-7 0z"/>
|
||||
@@ -145,7 +145,6 @@
|
||||
data-endpoint="{{ conn.endpoint_url }}"
|
||||
data-region="{{ conn.region }}"
|
||||
data-access="{{ conn.access_key }}"
|
||||
data-secret="{{ conn.secret_key }}"
|
||||
title="Edit connection">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" viewBox="0 0 16 16">
|
||||
<path d="M12.146.146a.5.5 0 0 1 .708 0l3 3a.5.5 0 0 1 0 .708l-10 10a.5.5 0 0 1-.168.11l-5 2a.5.5 0 0 1-.65-.65l2-5a.5.5 0 0 1 .11-.168l10-10zM11.207 2.5 13.5 4.793 14.793 3.5 12.5 1.207 11.207 2.5zm1.586 3L10.5 3.207 4 9.707V10h.5a.5.5 0 0 1 .5.5v.5h.5a.5.5 0 0 1 .5.5v.5h.293l6.5-6.5z"/>
|
||||
@@ -220,7 +219,7 @@
|
||||
<label for="edit_secret_key" class="form-label fw-medium">Secret Key</label>
|
||||
<div class="input-group">
|
||||
<input type="password" class="form-control font-monospace" id="edit_secret_key" name="secret_key" required>
|
||||
<button class="btn btn-outline-secondary" type="button" onclick="togglePassword('edit_secret_key')">
|
||||
<button class="btn btn-outline-secondary" type="button" onclick="ConnectionsManagement.togglePassword('edit_secret_key')">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" viewBox="0 0 16 16">
|
||||
<path d="M16 8s-3-5.5-8-5.5S0 8 0 8s3 5.5 8 5.5S16 8 16 8zM1.173 8a13.133 13.133 0 0 1 1.66-2.043C4.12 4.668 5.88 3.5 8 3.5c2.12 0 3.879 1.168 5.168 2.457A13.133 13.133 0 0 1 14.828 8c-.058.087-.122.183-.195.288-.335.48-.83 1.12-1.465 1.755C11.879 11.332 10.119 12.5 8 12.5c-2.12 0-3.879-1.168-5.168-2.457A13.134 13.134 0 0 1 1.172 8z"/>
|
||||
<path d="M8 5.5a2.5 2.5 0 1 0 0 5 2.5 2.5 0 0 0 0-5zM4.5 8a3.5 3.5 0 1 1 7 0 3.5 3.5 0 0 1-7 0z"/>
|
||||
@@ -289,153 +288,16 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script src="{{ url_for('static', filename='js/connections-management.js') }}"></script>
|
||||
<script>
|
||||
function togglePassword(id) {
|
||||
const input = document.getElementById(id);
|
||||
if (input.type === "password") {
|
||||
input.type = "text";
|
||||
} else {
|
||||
input.type = "password";
|
||||
}
|
||||
ConnectionsManagement.init({
|
||||
csrfToken: "{{ csrf_token() }}",
|
||||
endpoints: {
|
||||
test: "{{ url_for('ui.test_connection') }}",
|
||||
updateTemplate: "{{ url_for('ui.update_connection', connection_id='CONNECTION_ID') }}",
|
||||
deleteTemplate: "{{ url_for('ui.delete_connection', connection_id='CONNECTION_ID') }}",
|
||||
healthTemplate: "/ui/connections/CONNECTION_ID/health"
|
||||
}
|
||||
|
||||
async function testConnection(formId, resultId) {
|
||||
const form = document.getElementById(formId);
|
||||
const resultDiv = document.getElementById(resultId);
|
||||
const formData = new FormData(form);
|
||||
const data = Object.fromEntries(formData.entries());
|
||||
|
||||
resultDiv.innerHTML = '<div class="text-info"><span class="spinner-border spinner-border-sm" role="status" aria-hidden="true"></span> Testing connection...</div>';
|
||||
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), 20000);
|
||||
|
||||
try {
|
||||
const response = await fetch("{{ url_for('ui.test_connection') }}", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
"X-CSRFToken": "{{ csrf_token() }}"
|
||||
},
|
||||
body: JSON.stringify(data),
|
||||
signal: controller.signal
|
||||
});
|
||||
clearTimeout(timeoutId);
|
||||
|
||||
const result = await response.json();
|
||||
if (response.ok) {
|
||||
resultDiv.innerHTML = `<div class="text-success">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="me-1" viewBox="0 0 16 16">
|
||||
<path d="M16 8A8 8 0 1 1 0 8a8 8 0 0 1 16 0zm-3.97-3.03a.75.75 0 0 0-1.08.022L7.477 9.417 5.384 7.323a.75.75 0 0 0-1.06 1.06L6.97 11.03a.75.75 0 0 0 1.079-.02l3.992-4.99a.75.75 0 0 0-.01-1.05z"/>
|
||||
</svg>
|
||||
${result.message}
|
||||
</div>`;
|
||||
} else {
|
||||
resultDiv.innerHTML = `<div class="text-danger">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="me-1" viewBox="0 0 16 16">
|
||||
<path d="M16 8A8 8 0 1 1 0 8a8 8 0 0 1 16 0zM5.354 4.646a.5.5 0 1 0-.708.708L7.293 8l-2.647 2.646a.5.5 0 0 0 .708.708L8 8.707l2.646 2.647a.5.5 0 0 0 .708-.708L8.707 8l2.647-2.646a.5.5 0 0 0-.708-.708L8 7.293 5.354 4.646z"/>
|
||||
</svg>
|
||||
${result.message}
|
||||
</div>`;
|
||||
}
|
||||
} catch (error) {
|
||||
clearTimeout(timeoutId);
|
||||
if (error.name === 'AbortError') {
|
||||
resultDiv.innerHTML = `<div class="text-danger">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="me-1" viewBox="0 0 16 16">
|
||||
<path d="M16 8A8 8 0 1 1 0 8a8 8 0 0 1 16 0zM5.354 4.646a.5.5 0 1 0-.708.708L7.293 8l-2.647 2.646a.5.5 0 0 0 .708.708L8 8.707l2.646 2.647a.5.5 0 0 0 .708-.708L8.707 8l2.647-2.646a.5.5 0 0 0-.708-.708L8 7.293 5.354 4.646z"/>
|
||||
</svg>
|
||||
Connection test timed out - endpoint may be unreachable
|
||||
</div>`;
|
||||
} else {
|
||||
resultDiv.innerHTML = `<div class="text-danger">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="me-1" viewBox="0 0 16 16">
|
||||
<path d="M16 8A8 8 0 1 1 0 8a8 8 0 0 1 16 0zM5.354 4.646a.5.5 0 1 0-.708.708L7.293 8l-2.647 2.646a.5.5 0 0 0 .708.708L8 8.707l2.646 2.647a.5.5 0 0 0 .708-.708L8.707 8l2.647-2.646a.5.5 0 0 0-.708-.708L8 7.293 5.354 4.646z"/>
|
||||
</svg>
|
||||
Connection failed: Network error
|
||||
</div>`;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
document.getElementById('testConnectionBtn').addEventListener('click', () => {
|
||||
testConnection('createConnectionForm', 'testResult');
|
||||
});
|
||||
|
||||
document.getElementById('editTestConnectionBtn').addEventListener('click', () => {
|
||||
testConnection('editConnectionForm', 'editTestResult');
|
||||
});
|
||||
|
||||
const editModal = document.getElementById('editConnectionModal');
|
||||
editModal.addEventListener('show.bs.modal', event => {
|
||||
const button = event.relatedTarget;
|
||||
const id = button.getAttribute('data-id');
|
||||
|
||||
document.getElementById('edit_name').value = button.getAttribute('data-name');
|
||||
document.getElementById('edit_endpoint_url').value = button.getAttribute('data-endpoint');
|
||||
document.getElementById('edit_region').value = button.getAttribute('data-region');
|
||||
document.getElementById('edit_access_key').value = button.getAttribute('data-access');
|
||||
document.getElementById('edit_secret_key').value = button.getAttribute('data-secret');
|
||||
document.getElementById('editTestResult').innerHTML = '';
|
||||
|
||||
const form = document.getElementById('editConnectionForm');
|
||||
form.action = "{{ url_for('ui.update_connection', connection_id='CONN_ID') }}".replace('CONN_ID', id);
|
||||
});
|
||||
|
||||
const deleteModal = document.getElementById('deleteConnectionModal');
|
||||
deleteModal.addEventListener('show.bs.modal', event => {
|
||||
const button = event.relatedTarget;
|
||||
const id = button.getAttribute('data-id');
|
||||
const name = button.getAttribute('data-name');
|
||||
|
||||
document.getElementById('deleteConnectionName').textContent = name;
|
||||
const form = document.getElementById('deleteConnectionForm');
|
||||
form.action = "{{ url_for('ui.delete_connection', connection_id='CONN_ID') }}".replace('CONN_ID', id);
|
||||
});
|
||||
|
||||
async function checkConnectionHealth(connectionId, statusEl) {
|
||||
try {
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), 15000);
|
||||
|
||||
const response = await fetch(`/ui/connections/${connectionId}/health`, {
|
||||
signal: controller.signal
|
||||
});
|
||||
clearTimeout(timeoutId);
|
||||
|
||||
const data = await response.json();
|
||||
if (data.healthy) {
|
||||
statusEl.innerHTML = `
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="text-success" viewBox="0 0 16 16">
|
||||
<path d="M16 8A8 8 0 1 1 0 8a8 8 0 0 1 16 0zm-3.97-3.03a.75.75 0 0 0-1.08.022L7.477 9.417 5.384 7.323a.75.75 0 0 0-1.06 1.06L6.97 11.03a.75.75 0 0 0 1.079-.02l3.992-4.99a.75.75 0 0 0-.01-1.05z"/>
|
||||
</svg>`;
|
||||
statusEl.setAttribute('data-status', 'healthy');
|
||||
statusEl.setAttribute('title', 'Connected');
|
||||
} else {
|
||||
statusEl.innerHTML = `
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="text-danger" viewBox="0 0 16 16">
|
||||
<path d="M16 8A8 8 0 1 1 0 8a8 8 0 0 1 16 0zM5.354 4.646a.5.5 0 1 0-.708.708L7.293 8l-2.647 2.646a.5.5 0 0 0 .708.708L8 8.707l2.646 2.647a.5.5 0 0 0 .708-.708L8.707 8l2.647-2.646a.5.5 0 0 0-.708-.708L8 7.293 5.354 4.646z"/>
|
||||
</svg>`;
|
||||
statusEl.setAttribute('data-status', 'unhealthy');
|
||||
statusEl.setAttribute('title', data.error || 'Unreachable');
|
||||
}
|
||||
} catch (error) {
|
||||
statusEl.innerHTML = `
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="text-warning" viewBox="0 0 16 16">
|
||||
<path d="M8.982 1.566a1.13 1.13 0 0 0-1.96 0L.165 13.233c-.457.778.091 1.767.98 1.767h13.713c.889 0 1.438-.99.98-1.767L8.982 1.566zM8 5c.535 0 .954.462.9.995l-.35 3.507a.552.552 0 0 1-1.1 0L7.1 5.995A.905.905 0 0 1 8 5zm.002 6a1 1 0 1 1 0 2 1 1 0 0 1 0-2z"/>
|
||||
</svg>`;
|
||||
statusEl.setAttribute('data-status', 'unknown');
|
||||
statusEl.setAttribute('title', 'Could not check status');
|
||||
}
|
||||
}
|
||||
|
||||
const connectionRows = document.querySelectorAll('tr[data-connection-id]');
|
||||
connectionRows.forEach((row, index) => {
|
||||
const connectionId = row.getAttribute('data-connection-id');
|
||||
const statusEl = row.querySelector('.connection-status');
|
||||
if (statusEl) {
|
||||
setTimeout(() => checkConnectionHealth(connectionId, statusEl), index * 200);
|
||||
}
|
||||
});
|
||||
});
|
||||
</script>
|
||||
{% endblock %}
|
||||
|
||||
1711
templates/docs.html
1711
templates/docs.html
File diff suppressed because it is too large
Load Diff
@@ -110,24 +110,55 @@
|
||||
{% else %}
|
||||
<div class="card-body px-4 pb-4">
|
||||
{% if users %}
|
||||
{% if users|length > 1 %}
|
||||
<div class="mb-3">
|
||||
<div class="search-input-wrapper">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="search-icon" viewBox="0 0 16 16">
|
||||
<path d="M11.742 10.344a6.5 6.5 0 1 0-1.397 1.398h-.001c.03.04.062.078.098.115l3.85 3.85a1 1 0 0 0 1.415-1.414l-3.85-3.85a1.007 1.007 0 0 0-.115-.1zM12 6.5a5.5 5.5 0 1 1-11 0 5.5 5.5 0 0 1 11 0z"/>
|
||||
</svg>
|
||||
<input type="text" class="form-control" id="iam-user-search" placeholder="Filter users by name or access key..." autocomplete="off" />
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
<div class="row g-3">
|
||||
{% for user in users %}
|
||||
<div class="col-md-6 col-xl-4">
|
||||
<div class="card h-100 iam-user-card">
|
||||
{% set ns = namespace(is_admin=false) %}
|
||||
{% for policy in user.policies %}
|
||||
{% if 'iam:*' in policy.actions or '*' in policy.actions %}
|
||||
{% set ns.is_admin = true %}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
<div class="col-md-6 col-xl-4 iam-user-item" data-display-name="{{ user.display_name|lower }}" data-access-key-filter="{{ user.access_key|lower }}">
|
||||
<div class="card h-100 iam-user-card{{ ' iam-admin-card' if ns.is_admin else '' }}">
|
||||
<div class="card-body">
|
||||
<div class="d-flex align-items-start justify-content-between mb-3">
|
||||
<div class="d-flex align-items-center gap-3">
|
||||
<div class="user-avatar user-avatar-lg">
|
||||
<div class="d-flex align-items-center gap-3 min-width-0 overflow-hidden">
|
||||
<div class="user-avatar user-avatar-lg flex-shrink-0">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="currentColor" viewBox="0 0 16 16">
|
||||
<path d="M8 8a3 3 0 1 0 0-6 3 3 0 0 0 0 6zm2-3a2 2 0 1 1-4 0 2 2 0 0 1 4 0zm4 8c0 1-1 1-1 1H3s-1 0-1-1 1-4 6-4 6 3 6 4zm-1-.004c-.001-.246-.154-.986-.832-1.664C11.516 10.68 10.289 10 8 10c-2.29 0-3.516.68-4.168 1.332-.678.678-.83 1.418-.832 1.664h10z"/>
|
||||
</svg>
|
||||
</div>
|
||||
<div class="min-width-0">
|
||||
<h6 class="fw-semibold mb-0 text-truncate" title="{{ user.display_name }}">{{ user.display_name }}</h6>
|
||||
<code class="small text-muted d-block text-truncate" title="{{ user.access_key }}">{{ user.access_key }}</code>
|
||||
<div class="d-flex align-items-center gap-2 mb-0">
|
||||
<h6 class="fw-semibold mb-0 text-truncate" title="{{ user.display_name }}">{{ user.display_name }}</h6>
|
||||
{% if ns.is_admin %}
|
||||
<span class="iam-role-badge iam-role-admin" data-role-badge>Admin</span>
|
||||
{% else %}
|
||||
<span class="iam-role-badge iam-role-user" data-role-badge>User</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
<div class="d-flex align-items-center gap-1">
|
||||
<code class="small text-muted text-truncate" title="{{ user.access_key }}">{{ user.access_key }}</code>
|
||||
<button type="button" class="iam-copy-key" title="Copy access key" data-copy-access-key="{{ user.access_key }}">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="12" height="12" fill="currentColor" viewBox="0 0 16 16">
|
||||
<path d="M4 1.5H3a2 2 0 0 0-2 2V14a2 2 0 0 0 2 2h10a2 2 0 0 0 2-2V3.5a2 2 0 0 0-2-2h-1v1h1a1 1 0 0 1 1 1V14a1 1 0 0 1-1 1H3a1 1 0 0 1-1-1V3.5a1 1 0 0 1 1-1h1v-1z"/>
|
||||
<path d="M9.5 1a.5.5 0 0 1 .5.5v1a.5.5 0 0 1-.5.5h-3a.5.5 0 0 1-.5-.5v-1a.5.5 0 0 1 .5-.5h3zm-3-1A1.5 1.5 0 0 0 5 1.5v1A1.5 1.5 0 0 0 6.5 4h3A1.5 1.5 0 0 0 11 2.5v-1A1.5 1.5 0 0 0 9.5 0h-3z"/>
|
||||
</svg>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="dropdown">
|
||||
<div class="dropdown flex-shrink-0">
|
||||
<button class="btn btn-sm btn-icon" type="button" data-bs-toggle="dropdown" aria-expanded="false">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" viewBox="0 0 16 16">
|
||||
<path d="M9.5 13a1.5 1.5 0 1 1-3 0 1.5 1.5 0 0 1 3 0zm0-5a1.5 1.5 0 1 1-3 0 1.5 1.5 0 0 1 3 0zm0-5a1.5 1.5 0 1 1-3 0 1.5 1.5 0 0 1 3 0z"/>
|
||||
@@ -166,18 +197,27 @@
|
||||
</div>
|
||||
<div class="mb-3">
|
||||
<div class="small text-muted mb-2">Bucket Permissions</div>
|
||||
<div class="d-flex flex-wrap gap-1">
|
||||
<div class="d-flex flex-wrap gap-1" data-policy-badges>
|
||||
{% for policy in user.policies %}
|
||||
<span class="badge bg-primary bg-opacity-10 text-primary">
|
||||
{% set bucket_label = 'All Buckets' if policy.bucket == '*' else policy.bucket %}
|
||||
{% if '*' in policy.actions %}
|
||||
{% set perm_label = 'Full Access' %}
|
||||
{% elif policy.actions|length >= 9 %}
|
||||
{% set perm_label = 'Full Access' %}
|
||||
{% elif 'list' in policy.actions and 'read' in policy.actions and 'write' in policy.actions and 'delete' in policy.actions %}
|
||||
{% set perm_label = 'Read + Write + Delete' %}
|
||||
{% elif 'list' in policy.actions and 'read' in policy.actions and 'write' in policy.actions %}
|
||||
{% set perm_label = 'Read + Write' %}
|
||||
{% elif 'list' in policy.actions and 'read' in policy.actions %}
|
||||
{% set perm_label = 'Read Only' %}
|
||||
{% else %}
|
||||
{% set perm_label = 'Custom (' ~ policy.actions|length ~ ')' %}
|
||||
{% endif %}
|
||||
<span class="iam-perm-badge">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="10" height="10" fill="currentColor" class="me-1" viewBox="0 0 16 16">
|
||||
<path d="M2.522 5H2a.5.5 0 0 0-.494.574l1.372 9.149A1.5 1.5 0 0 0 4.36 16h7.278a1.5 1.5 0 0 0 1.483-1.277l1.373-9.149A.5.5 0 0 0 14 5h-.522A5.5 5.5 0 0 0 2.522 5zm1.005 0a4.5 4.5 0 0 1 8.945 0H3.527z"/>
|
||||
</svg>
|
||||
{{ policy.bucket }}
|
||||
{% if '*' in policy.actions %}
|
||||
<span class="opacity-75">(full)</span>
|
||||
{% else %}
|
||||
<span class="opacity-75">({{ policy.actions|length }})</span>
|
||||
{% endif %}
|
||||
{{ bucket_label }} · {{ perm_label }}
|
||||
</span>
|
||||
{% else %}
|
||||
<span class="badge bg-secondary bg-opacity-10 text-secondary">No policies</span>
|
||||
@@ -196,6 +236,12 @@
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
<div class="iam-no-results d-none" id="iam-no-results">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="currentColor" class="mb-2" viewBox="0 0 16 16">
|
||||
<path d="M11.742 10.344a6.5 6.5 0 1 0-1.397 1.398h-.001c.03.04.062.078.098.115l3.85 3.85a1 1 0 0 0 1.415-1.414l-3.85-3.85a1.007 1.007 0 0 0-.115-.1zM12 6.5a5.5 5.5 0 1 1-11 0 5.5 5.5 0 0 1 11 0z"/>
|
||||
</svg>
|
||||
<p class="mb-0">No users match your filter.</p>
|
||||
</div>
|
||||
{% else %}
|
||||
<div class="empty-state text-center py-5">
|
||||
<div class="empty-state-icon mx-auto mb-3">
|
||||
@@ -454,339 +500,20 @@
|
||||
|
||||
{% block extra_scripts %}
|
||||
{{ super() }}
|
||||
<script src="{{ url_for('static', filename='js/iam-management.js') }}"></script>
|
||||
<script>
|
||||
(function () {
|
||||
function setupJsonAutoIndent(textarea) {
|
||||
if (!textarea) return;
|
||||
|
||||
textarea.addEventListener('keydown', function(e) {
|
||||
if (e.key === 'Enter') {
|
||||
e.preventDefault();
|
||||
|
||||
const start = this.selectionStart;
|
||||
const end = this.selectionEnd;
|
||||
const value = this.value;
|
||||
|
||||
const lineStart = value.lastIndexOf('\n', start - 1) + 1;
|
||||
const currentLine = value.substring(lineStart, start);
|
||||
|
||||
const indentMatch = currentLine.match(/^(\s*)/);
|
||||
let indent = indentMatch ? indentMatch[1] : '';
|
||||
|
||||
const trimmedLine = currentLine.trim();
|
||||
const lastChar = trimmedLine.slice(-1);
|
||||
|
||||
const charBeforeCursor = value.substring(start - 1, start).trim();
|
||||
|
||||
let newIndent = indent;
|
||||
let insertAfter = '';
|
||||
|
||||
if (lastChar === '{' || lastChar === '[') {
|
||||
newIndent = indent + ' ';
|
||||
|
||||
const charAfterCursor = value.substring(start, start + 1).trim();
|
||||
if ((lastChar === '{' && charAfterCursor === '}') ||
|
||||
(lastChar === '[' && charAfterCursor === ']')) {
|
||||
insertAfter = '\n' + indent;
|
||||
}
|
||||
} else if (lastChar === ',' || lastChar === ':') {
|
||||
newIndent = indent;
|
||||
}
|
||||
|
||||
const insertion = '\n' + newIndent + insertAfter;
|
||||
const newValue = value.substring(0, start) + insertion + value.substring(end);
|
||||
|
||||
this.value = newValue;
|
||||
|
||||
const newCursorPos = start + 1 + newIndent.length;
|
||||
this.selectionStart = this.selectionEnd = newCursorPos;
|
||||
|
||||
this.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
}
|
||||
|
||||
if (e.key === 'Tab') {
|
||||
e.preventDefault();
|
||||
const start = this.selectionStart;
|
||||
const end = this.selectionEnd;
|
||||
|
||||
if (e.shiftKey) {
|
||||
const lineStart = this.value.lastIndexOf('\n', start - 1) + 1;
|
||||
const lineContent = this.value.substring(lineStart, start);
|
||||
if (lineContent.startsWith(' ')) {
|
||||
this.value = this.value.substring(0, lineStart) +
|
||||
this.value.substring(lineStart + 2);
|
||||
this.selectionStart = this.selectionEnd = Math.max(lineStart, start - 2);
|
||||
}
|
||||
} else {
|
||||
this.value = this.value.substring(0, start) + ' ' + this.value.substring(end);
|
||||
this.selectionStart = this.selectionEnd = start + 2;
|
||||
}
|
||||
|
||||
this.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
}
|
||||
});
|
||||
IAMManagement.init({
|
||||
users: JSON.parse(document.getElementById('iamUsersJson').textContent || '[]'),
|
||||
currentUserKey: {{ principal.access_key | tojson }},
|
||||
iamLocked: {{ iam_locked | tojson }},
|
||||
csrfToken: "{{ csrf_token() }}",
|
||||
endpoints: {
|
||||
createUser: "{{ url_for('ui.create_iam_user') }}",
|
||||
updateUser: "{{ url_for('ui.update_iam_user', access_key='ACCESS_KEY') }}",
|
||||
deleteUser: "{{ url_for('ui.delete_iam_user', access_key='ACCESS_KEY') }}",
|
||||
updatePolicies: "{{ url_for('ui.update_iam_policies', access_key='ACCESS_KEY') }}",
|
||||
rotateSecret: "{{ url_for('ui.rotate_iam_secret', access_key='ACCESS_KEY') }}"
|
||||
}
|
||||
|
||||
setupJsonAutoIndent(document.getElementById('policyEditorDocument'));
|
||||
setupJsonAutoIndent(document.getElementById('createUserPolicies'));
|
||||
|
||||
const currentUserKey = {{ principal.access_key | tojson }};
|
||||
const configCopyButtons = document.querySelectorAll('.config-copy');
|
||||
configCopyButtons.forEach((button) => {
|
||||
button.addEventListener('click', async () => {
|
||||
const targetId = button.dataset.copyTarget;
|
||||
const target = document.getElementById(targetId);
|
||||
if (!target) return;
|
||||
const text = target.innerText;
|
||||
try {
|
||||
await navigator.clipboard.writeText(text);
|
||||
button.textContent = 'Copied!';
|
||||
setTimeout(() => {
|
||||
button.textContent = 'Copy JSON';
|
||||
}, 1500);
|
||||
} catch (err) {
|
||||
console.error('Unable to copy IAM config', err);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
const secretCopyButton = document.querySelector('[data-secret-copy]');
|
||||
if (secretCopyButton) {
|
||||
secretCopyButton.addEventListener('click', async () => {
|
||||
const secretInput = document.getElementById('disclosedSecretValue');
|
||||
if (!secretInput) return;
|
||||
try {
|
||||
await navigator.clipboard.writeText(secretInput.value);
|
||||
secretCopyButton.textContent = 'Copied!';
|
||||
setTimeout(() => {
|
||||
secretCopyButton.textContent = 'Copy';
|
||||
}, 1500);
|
||||
} catch (err) {
|
||||
console.error('Unable to copy IAM secret', err);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
const iamUsersData = document.getElementById('iamUsersJson');
|
||||
const users = iamUsersData ? JSON.parse(iamUsersData.textContent || '[]') : [];
|
||||
|
||||
const policyModalEl = document.getElementById('policyEditorModal');
|
||||
const policyModal = new bootstrap.Modal(policyModalEl);
|
||||
const userLabelEl = document.getElementById('policyEditorUserLabel');
|
||||
const userInputEl = document.getElementById('policyEditorUser');
|
||||
const textareaEl = document.getElementById('policyEditorDocument');
|
||||
const formEl = document.getElementById('policyEditorForm');
|
||||
const templateButtons = document.querySelectorAll('[data-policy-template]');
|
||||
const iamLocked = {{ iam_locked | tojson }};
|
||||
|
||||
if (iamLocked) return;
|
||||
|
||||
const userPolicies = (accessKey) => {
|
||||
const target = users.find((user) => user.access_key === accessKey);
|
||||
return target ? JSON.stringify(target.policies, null, 2) : '';
|
||||
};
|
||||
|
||||
const applyTemplate = (name) => {
|
||||
const templates = {
|
||||
full: [
|
||||
{
|
||||
bucket: '*',
|
||||
actions: ['list', 'read', 'write', 'delete', 'share', 'policy', 'replication', 'iam:list_users', 'iam:*'],
|
||||
},
|
||||
],
|
||||
readonly: [
|
||||
{
|
||||
bucket: '*',
|
||||
actions: ['list', 'read'],
|
||||
},
|
||||
],
|
||||
writer: [
|
||||
{
|
||||
bucket: '*',
|
||||
actions: ['list', 'read', 'write'],
|
||||
},
|
||||
],
|
||||
};
|
||||
if (templates[name]) {
|
||||
textareaEl.value = JSON.stringify(templates[name], null, 2);
|
||||
}
|
||||
};
|
||||
|
||||
templateButtons.forEach((button) => {
|
||||
button.addEventListener('click', () => applyTemplate(button.dataset.policyTemplate));
|
||||
});
|
||||
|
||||
const createUserPoliciesEl = document.getElementById('createUserPolicies');
|
||||
const createTemplateButtons = document.querySelectorAll('[data-create-policy-template]');
|
||||
|
||||
const applyCreateTemplate = (name) => {
|
||||
const templates = {
|
||||
full: [
|
||||
{
|
||||
bucket: '*',
|
||||
actions: ['list', 'read', 'write', 'delete', 'share', 'policy', 'replication', 'iam:list_users', 'iam:*'],
|
||||
},
|
||||
],
|
||||
readonly: [
|
||||
{
|
||||
bucket: '*',
|
||||
actions: ['list', 'read'],
|
||||
},
|
||||
],
|
||||
writer: [
|
||||
{
|
||||
bucket: '*',
|
||||
actions: ['list', 'read', 'write'],
|
||||
},
|
||||
],
|
||||
};
|
||||
if (templates[name] && createUserPoliciesEl) {
|
||||
createUserPoliciesEl.value = JSON.stringify(templates[name], null, 2);
|
||||
}
|
||||
};
|
||||
|
||||
createTemplateButtons.forEach((button) => {
|
||||
button.addEventListener('click', () => applyCreateTemplate(button.dataset.createPolicyTemplate));
|
||||
});
|
||||
|
||||
formEl?.addEventListener('submit', (event) => {
|
||||
const key = userInputEl.value;
|
||||
if (!key) {
|
||||
event.preventDefault();
|
||||
return;
|
||||
}
|
||||
const template = formEl.dataset.actionTemplate;
|
||||
formEl.action = template.replace('ACCESS_KEY_PLACEHOLDER', key);
|
||||
});
|
||||
|
||||
document.querySelectorAll('[data-policy-editor]').forEach((button) => {
|
||||
button.addEventListener('click', () => {
|
||||
const key = button.getAttribute('data-access-key');
|
||||
if (!key) return;
|
||||
|
||||
userLabelEl.textContent = key;
|
||||
userInputEl.value = key;
|
||||
textareaEl.value = userPolicies(key);
|
||||
|
||||
policyModal.show();
|
||||
});
|
||||
});
|
||||
|
||||
const editUserModal = new bootstrap.Modal(document.getElementById('editUserModal'));
|
||||
const editUserForm = document.getElementById('editUserForm');
|
||||
const editUserDisplayName = document.getElementById('editUserDisplayName');
|
||||
|
||||
document.querySelectorAll('[data-edit-user]').forEach(btn => {
|
||||
btn.addEventListener('click', () => {
|
||||
const key = btn.dataset.editUser;
|
||||
const name = btn.dataset.displayName;
|
||||
editUserDisplayName.value = name;
|
||||
editUserForm.action = "{{ url_for('ui.update_iam_user', access_key='ACCESS_KEY') }}".replace('ACCESS_KEY', key);
|
||||
editUserModal.show();
|
||||
});
|
||||
});
|
||||
|
||||
const deleteUserModal = new bootstrap.Modal(document.getElementById('deleteUserModal'));
|
||||
const deleteUserForm = document.getElementById('deleteUserForm');
|
||||
const deleteUserLabel = document.getElementById('deleteUserLabel');
|
||||
const deleteSelfWarning = document.getElementById('deleteSelfWarning');
|
||||
|
||||
document.querySelectorAll('[data-delete-user]').forEach(btn => {
|
||||
btn.addEventListener('click', () => {
|
||||
const key = btn.dataset.deleteUser;
|
||||
deleteUserLabel.textContent = key;
|
||||
deleteUserForm.action = "{{ url_for('ui.delete_iam_user', access_key='ACCESS_KEY') }}".replace('ACCESS_KEY', key);
|
||||
|
||||
if (key === currentUserKey) {
|
||||
deleteSelfWarning.classList.remove('d-none');
|
||||
} else {
|
||||
deleteSelfWarning.classList.add('d-none');
|
||||
}
|
||||
|
||||
deleteUserModal.show();
|
||||
});
|
||||
});
|
||||
|
||||
const rotateSecretModal = new bootstrap.Modal(document.getElementById('rotateSecretModal'));
|
||||
const rotateUserLabel = document.getElementById('rotateUserLabel');
|
||||
const confirmRotateBtn = document.getElementById('confirmRotateBtn');
|
||||
const rotateCancelBtn = document.getElementById('rotateCancelBtn');
|
||||
const rotateDoneBtn = document.getElementById('rotateDoneBtn');
|
||||
const rotateSecretConfirm = document.getElementById('rotateSecretConfirm');
|
||||
const rotateSecretResult = document.getElementById('rotateSecretResult');
|
||||
const newSecretKeyInput = document.getElementById('newSecretKey');
|
||||
const copyNewSecretBtn = document.getElementById('copyNewSecret');
|
||||
let currentRotateKey = null;
|
||||
|
||||
document.querySelectorAll('[data-rotate-user]').forEach(btn => {
|
||||
btn.addEventListener('click', () => {
|
||||
currentRotateKey = btn.dataset.rotateUser;
|
||||
rotateUserLabel.textContent = currentRotateKey;
|
||||
|
||||
rotateSecretConfirm.classList.remove('d-none');
|
||||
rotateSecretResult.classList.add('d-none');
|
||||
confirmRotateBtn.classList.remove('d-none');
|
||||
rotateCancelBtn.classList.remove('d-none');
|
||||
rotateDoneBtn.classList.add('d-none');
|
||||
|
||||
rotateSecretModal.show();
|
||||
});
|
||||
});
|
||||
|
||||
confirmRotateBtn.addEventListener('click', async () => {
|
||||
if (!currentRotateKey) return;
|
||||
|
||||
confirmRotateBtn.disabled = true;
|
||||
confirmRotateBtn.textContent = "Rotating...";
|
||||
|
||||
try {
|
||||
const url = "{{ url_for('ui.rotate_iam_secret', access_key='ACCESS_KEY') }}".replace('ACCESS_KEY', currentRotateKey);
|
||||
const response = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Accept': 'application/json',
|
||||
'X-CSRFToken': "{{ csrf_token() }}"
|
||||
}
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const data = await response.json();
|
||||
throw new Error(data.error || 'Failed to rotate secret');
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
newSecretKeyInput.value = data.secret_key;
|
||||
|
||||
rotateSecretConfirm.classList.add('d-none');
|
||||
rotateSecretResult.classList.remove('d-none');
|
||||
confirmRotateBtn.classList.add('d-none');
|
||||
rotateCancelBtn.classList.add('d-none');
|
||||
rotateDoneBtn.classList.remove('d-none');
|
||||
|
||||
} catch (err) {
|
||||
if (window.showToast) {
|
||||
window.showToast(err.message, 'Error', 'danger');
|
||||
}
|
||||
rotateSecretModal.hide();
|
||||
} finally {
|
||||
confirmRotateBtn.disabled = false;
|
||||
confirmRotateBtn.textContent = "Rotate Key";
|
||||
}
|
||||
});
|
||||
|
||||
copyNewSecretBtn.addEventListener('click', async () => {
|
||||
try {
|
||||
await navigator.clipboard.writeText(newSecretKeyInput.value);
|
||||
copyNewSecretBtn.textContent = 'Copied!';
|
||||
setTimeout(() => copyNewSecretBtn.textContent = 'Copy', 1500);
|
||||
} catch (err) {
|
||||
console.error('Failed to copy', err);
|
||||
}
|
||||
});
|
||||
|
||||
rotateDoneBtn.addEventListener('click', () => {
|
||||
window.location.reload();
|
||||
});
|
||||
})();
|
||||
});
|
||||
</script>
|
||||
{% endblock %}
|
||||
|
||||
@@ -74,7 +74,7 @@
|
||||
</button>
|
||||
</form>
|
||||
<div class="text-center mt-4">
|
||||
<small class="text-muted">Need help? Check the <a href="#" class="text-decoration-none">documentation</a></small>
|
||||
<small class="text-muted">Need help? Check the <a href="{{ url_for('ui.docs_page') }}" class="text-decoration-none">documentation</a></small>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
270
templates/replication_wizard.html
Normal file
270
templates/replication_wizard.html
Normal file
@@ -0,0 +1,270 @@
|
||||
{% extends "base.html" %}
|
||||
|
||||
{% block title %}Set Up Replication - S3 Compatible Storage{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<div class="page-header d-flex justify-content-between align-items-center mb-4">
|
||||
<div>
|
||||
<nav aria-label="breadcrumb">
|
||||
<ol class="breadcrumb mb-1">
|
||||
<li class="breadcrumb-item"><a href="{{ url_for('ui.sites_dashboard') }}">Sites</a></li>
|
||||
<li class="breadcrumb-item active" aria-current="page">Replication Wizard</li>
|
||||
</ol>
|
||||
</nav>
|
||||
<h1 class="h3 mb-1 d-flex align-items-center gap-2">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="28" height="28" fill="currentColor" class="text-primary" viewBox="0 0 16 16">
|
||||
<path d="M8 4a.5.5 0 0 1 .5.5V6a.5.5 0 0 1-1 0V4.5A.5.5 0 0 1 8 4zM3.732 5.732a.5.5 0 0 1 .707 0l.915.914a.5.5 0 1 1-.708.708l-.914-.915a.5.5 0 0 1 0-.707zM2 10a.5.5 0 0 1 .5-.5h1.586a.5.5 0 0 1 0 1H2.5A.5.5 0 0 1 2 10zm9.5 0a.5.5 0 0 1 .5-.5h1.5a.5.5 0 0 1 0 1H12a.5.5 0 0 1-.5-.5zm.754-4.246a.389.389 0 0 0-.527-.02L7.547 9.31a.91.91 0 1 0 1.302 1.258l3.434-4.297a.389.389 0 0 0-.029-.518z"/>
|
||||
<path fill-rule="evenodd" d="M0 10a8 8 0 1 1 15.547 2.661c-.442 1.253-1.845 1.602-2.932 1.25C11.309 13.488 9.475 13 8 13c-1.474 0-3.31.488-4.615.911-1.087.352-2.49.003-2.932-1.25A7.988 7.988 0 0 1 0 10zm8-7a7 7 0 0 0-6.603 9.329c.203.575.923.876 1.68.63C4.397 12.533 6.358 12 8 12s3.604.532 4.923.96c.757.245 1.477-.056 1.68-.631A7 7 0 0 0 8 3z"/>
|
||||
</svg>
|
||||
Set Up Replication
|
||||
</h1>
|
||||
<p class="text-muted mb-0 mt-1">Configure bucket replication to <strong>{{ peer.display_name or peer.site_id }}</strong></p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="row g-4">
|
||||
<div class="col-lg-4 col-md-5">
|
||||
<div class="card shadow-sm border-0 mb-4" style="border-radius: 1rem;">
|
||||
<div class="card-header bg-transparent border-0 pt-4 pb-0 px-4">
|
||||
<h5 class="fw-semibold d-flex align-items-center gap-2 mb-1">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" class="text-primary" viewBox="0 0 16 16">
|
||||
<path d="M0 8a8 8 0 1 1 16 0A8 8 0 0 1 0 8m7.5-6.923c-.67.204-1.335.82-1.887 1.855A8 8 0 0 0 5.145 4H7.5zM4.09 4a9.3 9.3 0 0 1 .64-1.539 7 7 0 0 1 .597-.933A7.03 7.03 0 0 0 2.255 4zm-.582 3.5c.03-.877.138-1.718.312-2.5H1.674a7 7 0 0 0-.656 2.5zM4.847 5a12.5 12.5 0 0 0-.338 2.5H7.5V5zM8.5 5v2.5h2.99a12.5 12.5 0 0 0-.337-2.5zM4.51 8.5a12.5 12.5 0 0 0 .337 2.5H7.5V8.5zm3.99 0V11h2.653c.187-.765.306-1.608.338-2.5zM5.145 12q.208.58.468 1.068c.552 1.035 1.218 1.65 1.887 1.855V12zm.182 2.472a7 7 0 0 1-.597-.933A9.3 9.3 0 0 1 4.09 12H2.255a7 7 0 0 0 3.072 2.472M3.82 11a13.7 13.7 0 0 1-.312-2.5h-2.49a7 7 0 0 0 .656 2.5zM8.5 12v2.923c.67-.204 1.335-.82 1.887-1.855q.26-.487.468-1.068zm3.68-1h2.146c.365-.767.594-1.61.656-2.5h-2.49a13.7 13.7 0 0 1-.312 2.5m2.802-3.5a7 7 0 0 0-.656-2.5H12.18c.174.782.282 1.623.312 2.5zM11.27 2.461c.247.464.462.98.64 1.539h1.835a7 7 0 0 0-3.072-2.472c.218.284.418.598.597.933M10.855 4a8 8 0 0 0-.468-1.068C9.835 1.897 9.17 1.282 8.5 1.077V4z"/>
|
||||
</svg>
|
||||
Peer Site
|
||||
</h5>
|
||||
</div>
|
||||
<div class="card-body px-4 pb-4">
|
||||
<dl class="mb-0">
|
||||
<dt class="text-muted small">Site ID</dt>
|
||||
<dd class="mb-2">{{ peer.site_id }}</dd>
|
||||
<dt class="text-muted small">Endpoint</dt>
|
||||
<dd class="mb-2 text-truncate" title="{{ peer.endpoint }}">{{ peer.endpoint }}</dd>
|
||||
<dt class="text-muted small">Region</dt>
|
||||
<dd class="mb-2"><span class="badge bg-primary bg-opacity-10 text-primary">{{ peer.region }}</span></dd>
|
||||
<dt class="text-muted small">Connection</dt>
|
||||
<dd class="mb-0"><span class="badge bg-secondary bg-opacity-10 text-secondary">{{ connection.name }}</span></dd>
|
||||
</dl>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card shadow-sm border-0" style="border-radius: 1rem;">
|
||||
<div class="card-header bg-transparent border-0 pt-4 pb-0 px-4">
|
||||
<h5 class="fw-semibold d-flex align-items-center gap-2 mb-1">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" class="text-muted" viewBox="0 0 16 16">
|
||||
<path d="M8 16A8 8 0 1 0 8 0a8 8 0 0 0 0 16zm.93-9.412-1 4.705c-.07.34.029.533.304.533.194 0 .487-.07.686-.246l-.088.416c-.287.346-.92.598-1.465.598-.703 0-1.002-.422-.808-1.319l.738-3.468c.064-.293.006-.399-.287-.47l-.451-.081.082-.381 2.29-.287zM8 5.5a1 1 0 1 1 0-2 1 1 0 0 1 0 2z"/>
|
||||
</svg>
|
||||
Replication Modes
|
||||
</h5>
|
||||
</div>
|
||||
<div class="card-body px-4 pb-4 small">
|
||||
<p class="mb-2"><strong>New Only:</strong> Only replicate new objects uploaded after the rule is created.</p>
|
||||
<p class="mb-2"><strong>All Objects:</strong> Replicate all existing objects plus new uploads.</p>
|
||||
<p class="mb-0"><strong>Bidirectional:</strong> Two-way sync between sites. Changes on either side are synchronized.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="col-lg-8 col-md-7">
|
||||
<div class="card shadow-sm border-0" style="border-radius: 1rem;">
|
||||
<div class="card-header bg-transparent border-0 pt-4 pb-0 px-4">
|
||||
<h5 class="fw-semibold d-flex align-items-center gap-2 mb-1">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" class="text-muted" viewBox="0 0 16 16">
|
||||
<path d="M2.522 5H2a.5.5 0 0 0-.494.574l1.372 9.149A1.5 1.5 0 0 0 4.36 16h7.278a1.5 1.5 0 0 0 1.483-1.277l1.373-9.149A.5.5 0 0 0 14 5h-.522A5.5 5.5 0 0 0 2.522 5zm1.005 0a4.5 4.5 0 0 1 8.945 0H3.527z"/>
|
||||
</svg>
|
||||
Select Buckets to Replicate
|
||||
</h5>
|
||||
<p class="text-muted small mb-0">Choose which buckets should be replicated to this peer site</p>
|
||||
</div>
|
||||
<div class="card-body px-4 pb-4">
|
||||
{% if buckets %}
|
||||
<form method="POST" action="{{ url_for('ui.create_peer_replication_rules', site_id=peer.site_id) }}">
|
||||
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
|
||||
|
||||
<div class="mb-4">
|
||||
<label for="mode" class="form-label fw-medium">Replication Mode</label>
|
||||
<select class="form-select" id="mode" name="mode">
|
||||
<option value="new_only">New Objects Only</option>
|
||||
<option value="all">All Objects (includes existing)</option>
|
||||
<option value="bidirectional">Bidirectional Sync</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<div id="bidirWarning" class="alert alert-warning d-none mb-4" role="alert">
|
||||
<h6 class="alert-heading fw-bold d-flex align-items-center gap-2">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" viewBox="0 0 16 16">
|
||||
<path fill-rule="evenodd" d="M1 11.5a.5.5 0 0 0 .5.5h11.793l-3.147 3.146a.5.5 0 0 0 .708.708l4-4a.5.5 0 0 0 0-.708l-4-4a.5.5 0 0 0-.708.708L13.293 11H1.5a.5.5 0 0 0-.5.5zm14-7a.5.5 0 0 1-.5.5H2.707l3.147 3.146a.5.5 0 1 1-.708.708l-4-4a.5.5 0 0 1 0-.708l4-4a.5.5 0 1 1 .708.708L2.707 4H14.5a.5.5 0 0 1 .5.5z"/>
|
||||
</svg>
|
||||
Bidirectional Sync Requires Configuration on Both Sites
|
||||
</h6>
|
||||
<p class="mb-2">For bidirectional sync to work properly, you must configure <strong>both</strong> sites. This wizard only configures one direction.</p>
|
||||
<hr class="my-2">
|
||||
<p class="mb-2 fw-semibold">After completing this wizard, you must also:</p>
|
||||
<ol class="mb-2 ps-3">
|
||||
<li>Go to <strong>{{ peer.display_name or peer.site_id }}</strong>'s admin UI</li>
|
||||
<li>Register <strong>this site</strong> as a peer (with a connection)</li>
|
||||
<li>Create matching bidirectional replication rules pointing back to this site</li>
|
||||
<li>Ensure <code>SITE_SYNC_ENABLED=true</code> is set on both sites</li>
|
||||
</ol>
|
||||
<div class="d-flex align-items-center gap-2 mt-3">
|
||||
<span class="badge bg-light text-dark border">Local Site ID: <strong>{{ local_site.site_id if local_site else 'Not configured' }}</strong></span>
|
||||
<span class="badge bg-light text-dark border">Local Endpoint: <strong>{{ local_site.endpoint if local_site and local_site.endpoint else 'Not configured' }}</strong></span>
|
||||
</div>
|
||||
{% if not local_site or not local_site.site_id or not local_site.endpoint %}
|
||||
<div class="alert alert-danger mt-3 mb-0 py-2">
|
||||
<small><strong>Warning:</strong> Your local site identity is not fully configured. The remote site won't be able to connect back. <a href="{{ url_for('ui.sites_dashboard') }}">Configure it now</a>.</small>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
<div class="table-responsive">
|
||||
<table class="table table-hover align-middle mb-0">
|
||||
<thead class="table-light">
|
||||
<tr>
|
||||
<th scope="col" style="width: 40px;">
|
||||
<input type="checkbox" class="form-check-input" id="selectAll">
|
||||
</th>
|
||||
<th scope="col">Local Bucket</th>
|
||||
<th scope="col">Target Bucket Name</th>
|
||||
<th scope="col">Status</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for bucket in buckets %}
|
||||
<tr>
|
||||
<td>
|
||||
<input type="checkbox" class="form-check-input bucket-checkbox"
|
||||
name="buckets" value="{{ bucket.name }}"
|
||||
{% if bucket.has_rule %}disabled{% endif %}>
|
||||
</td>
|
||||
<td>
|
||||
<div class="d-flex align-items-center gap-2">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="text-muted" viewBox="0 0 16 16">
|
||||
<path d="M2.522 5H2a.5.5 0 0 0-.494.574l1.372 9.149A1.5 1.5 0 0 0 4.36 16h7.278a1.5 1.5 0 0 0 1.483-1.277l1.373-9.149A.5.5 0 0 0 14 5h-.522A5.5 5.5 0 0 0 2.522 5zm1.005 0a4.5 4.5 0 0 1 8.945 0H3.527z"/>
|
||||
</svg>
|
||||
<span class="fw-medium">{{ bucket.name }}</span>
|
||||
</div>
|
||||
</td>
|
||||
<td>
|
||||
<input type="text" class="form-control form-control-sm"
|
||||
name="target_{{ bucket.name }}"
|
||||
value="{{ bucket.existing_target or bucket.name }}"
|
||||
placeholder="{{ bucket.name }}"
|
||||
{% if bucket.has_rule %}disabled{% endif %}>
|
||||
</td>
|
||||
<td>
|
||||
{% if bucket.has_rule %}
|
||||
<span class="badge bg-info bg-opacity-10 text-info">
|
||||
Already configured ({{ bucket.existing_mode }})
|
||||
</span>
|
||||
{% else %}
|
||||
<span class="badge bg-secondary bg-opacity-10 text-secondary">
|
||||
Not configured
|
||||
</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<div class="d-flex gap-2 mt-4 pt-3 border-top">
|
||||
<button type="submit" class="btn btn-primary" id="submitBtn" disabled>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-1" viewBox="0 0 16 16">
|
||||
<path d="M10.97 4.97a.75.75 0 0 1 1.07 1.05l-3.99 4.99a.75.75 0 0 1-1.08.02L4.324 8.384a.75.75 0 1 1 1.06-1.06l2.094 2.093 3.473-4.425a.267.267 0 0 1 .02-.022z"/>
|
||||
</svg>
|
||||
Create Replication Rules
|
||||
</button>
|
||||
<a href="{{ url_for('ui.sites_dashboard') }}" class="btn btn-outline-secondary">
|
||||
Skip for Now
|
||||
</a>
|
||||
</div>
|
||||
</form>
|
||||
{% else %}
|
||||
<div class="empty-state text-center py-5">
|
||||
<div class="empty-state-icon mx-auto mb-3">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="48" height="48" fill="currentColor" viewBox="0 0 16 16">
|
||||
<path d="M2.522 5H2a.5.5 0 0 0-.494.574l1.372 9.149A1.5 1.5 0 0 0 4.36 16h7.278a1.5 1.5 0 0 0 1.483-1.277l1.373-9.149A.5.5 0 0 0 14 5h-.522A5.5 5.5 0 0 0 2.522 5zm1.005 0a4.5 4.5 0 0 1 8.945 0H3.527z"/>
|
||||
</svg>
|
||||
</div>
|
||||
<h5 class="fw-semibold mb-2">No buckets yet</h5>
|
||||
<p class="text-muted mb-3">Create some buckets first, then come back to set up replication.</p>
|
||||
<a href="{{ url_for('ui.buckets_overview') }}" class="btn btn-primary">
|
||||
Go to Buckets
|
||||
</a>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
(function() {
|
||||
const selectAllCheckbox = document.getElementById('selectAll');
|
||||
const bucketCheckboxes = document.querySelectorAll('.bucket-checkbox:not(:disabled)');
|
||||
const submitBtn = document.getElementById('submitBtn');
|
||||
const modeSelect = document.getElementById('mode');
|
||||
const bidirWarning = document.getElementById('bidirWarning');
|
||||
|
||||
function updateBidirWarning() {
|
||||
if (modeSelect && bidirWarning) {
|
||||
if (modeSelect.value === 'bidirectional') {
|
||||
bidirWarning.classList.remove('d-none');
|
||||
} else {
|
||||
bidirWarning.classList.add('d-none');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (modeSelect) {
|
||||
modeSelect.addEventListener('change', updateBidirWarning);
|
||||
updateBidirWarning();
|
||||
}
|
||||
|
||||
function updateSubmitButton() {
|
||||
const checkedCount = document.querySelectorAll('.bucket-checkbox:checked').length;
|
||||
if (submitBtn) {
|
||||
submitBtn.disabled = checkedCount === 0;
|
||||
const text = checkedCount > 0
|
||||
? `Create ${checkedCount} Replication Rule${checkedCount > 1 ? 's' : ''}`
|
||||
: 'Create Replication Rules';
|
||||
submitBtn.innerHTML = `
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-1" viewBox="0 0 16 16">
|
||||
<path d="M10.97 4.97a.75.75 0 0 1 1.07 1.05l-3.99 4.99a.75.75 0 0 1-1.08.02L4.324 8.384a.75.75 0 1 1 1.06-1.06l2.094 2.093 3.473-4.425a.267.267 0 0 1 .02-.022z"/>
|
||||
</svg>
|
||||
${text}
|
||||
`;
|
||||
}
|
||||
}
|
||||
|
||||
function updateSelectAll() {
|
||||
if (selectAllCheckbox && bucketCheckboxes.length > 0) {
|
||||
const allChecked = Array.from(bucketCheckboxes).every(cb => cb.checked);
|
||||
const someChecked = Array.from(bucketCheckboxes).some(cb => cb.checked);
|
||||
selectAllCheckbox.checked = allChecked;
|
||||
selectAllCheckbox.indeterminate = someChecked && !allChecked;
|
||||
}
|
||||
}
|
||||
|
||||
if (selectAllCheckbox) {
|
||||
selectAllCheckbox.addEventListener('change', function() {
|
||||
bucketCheckboxes.forEach(cb => {
|
||||
cb.checked = this.checked;
|
||||
});
|
||||
updateSubmitButton();
|
||||
});
|
||||
}
|
||||
|
||||
bucketCheckboxes.forEach(cb => {
|
||||
cb.addEventListener('change', function() {
|
||||
updateSelectAll();
|
||||
updateSubmitButton();
|
||||
});
|
||||
});
|
||||
|
||||
updateSelectAll();
|
||||
updateSubmitButton();
|
||||
})();
|
||||
</script>
|
||||
{% endblock %}
|
||||
891
templates/sites.html
Normal file
891
templates/sites.html
Normal file
@@ -0,0 +1,891 @@
|
||||
{% extends "base.html" %}
|
||||
|
||||
{% block title %}Sites - S3 Compatible Storage{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<div class="page-header d-flex justify-content-between align-items-center mb-4">
|
||||
<div>
|
||||
<p class="text-uppercase text-muted small mb-1">Geo-Distribution</p>
|
||||
<h1 class="h3 mb-1 d-flex align-items-center gap-2">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="28" height="28" fill="currentColor" class="text-primary" viewBox="0 0 16 16">
|
||||
<path d="M0 8a8 8 0 1 1 16 0A8 8 0 0 1 0 8zm7.5-6.923c-.67.204-1.335.82-1.887 1.855A7.97 7.97 0 0 0 5.145 4H7.5V1.077zM4.09 4a9.267 9.267 0 0 1 .64-1.539 6.7 6.7 0 0 1 .597-.933A7.025 7.025 0 0 0 2.255 4H4.09zm-.582 3.5c.03-.877.138-1.718.312-2.5H1.674a6.958 6.958 0 0 0-.656 2.5h2.49zM4.847 5a12.5 12.5 0 0 0-.338 2.5H7.5V5H4.847zM8.5 5v2.5h2.99a12.495 12.495 0 0 0-.337-2.5H8.5zM4.51 8.5a12.5 12.5 0 0 0 .337 2.5H7.5V8.5H4.51zm3.99 0V11h2.653c.187-.765.306-1.608.338-2.5H8.5zM5.145 12c.138.386.295.744.468 1.068.552 1.035 1.218 1.65 1.887 1.855V12H5.145zm.182 2.472a6.696 6.696 0 0 1-.597-.933A9.268 9.268 0 0 1 4.09 12H2.255a7.024 7.024 0 0 0 3.072 2.472zM3.82 11a13.652 13.652 0 0 1-.312-2.5h-2.49c.062.89.291 1.733.656 2.5H3.82zm6.853 3.472A7.024 7.024 0 0 0 13.745 12H11.91a9.27 9.27 0 0 1-.64 1.539 6.688 6.688 0 0 1-.597.933zM8.5 12v2.923c.67-.204 1.335-.82 1.887-1.855.173-.324.33-.682.468-1.068H8.5zm3.68-1h2.146c.365-.767.594-1.61.656-2.5h-2.49a13.65 13.65 0 0 1-.312 2.5zm2.802-3.5a6.959 6.959 0 0 0-.656-2.5H12.18c.174.782.282 1.623.312 2.5h2.49zM11.27 2.461c.247.464.462.98.64 1.539h1.835a7.024 7.024 0 0 0-3.072-2.472c.218.284.418.598.597.933zM10.855 4a7.966 7.966 0 0 0-.468-1.068C9.835 1.897 9.17 1.282 8.5 1.077V4h2.355z"/>
|
||||
</svg>
|
||||
Site Registry
|
||||
</h1>
|
||||
<p class="text-muted mb-0 mt-1">Configure this site's identity and manage peer sites for geo-distribution.</p>
|
||||
</div>
|
||||
<div class="d-none d-md-flex align-items-center gap-2">
|
||||
{% if local_site and local_site.site_id %}
|
||||
<span class="badge bg-secondary bg-opacity-10 text-secondary fs-6 px-3 py-2">
|
||||
{{ local_site.site_id }}
|
||||
</span>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="text-muted" viewBox="0 0 16 16">
|
||||
<path fill-rule="evenodd" d="M1 8a.5.5 0 0 1 .5-.5h11.793l-3.147-3.146a.5.5 0 0 1 .708-.708l4 4a.5.5 0 0 1 0 .708l-4 4a.5.5 0 0 1-.708-.708L13.293 8.5H1.5A.5.5 0 0 1 1 8z"/>
|
||||
</svg>
|
||||
{% endif %}
|
||||
<span class="badge bg-primary bg-opacity-10 text-primary fs-6 px-3 py-2">
|
||||
{{ peers|length }} peer{{ 's' if peers|length != 1 else '' }}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="row g-4">
|
||||
<div class="col-lg-4 col-md-5">
|
||||
<div class="card shadow-sm border-0 mb-4" style="border-radius: 1rem;">
|
||||
<div class="card-header bg-transparent border-0 pt-4 pb-0 px-4">
|
||||
<h5 class="fw-semibold d-flex align-items-center gap-2 mb-1">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" class="text-primary" viewBox="0 0 16 16">
|
||||
<path d="M8 16s6-5.686 6-10A6 6 0 0 0 2 6c0 4.314 6 10 6 10zm0-7a3 3 0 1 1 0-6 3 3 0 0 1 0 6z"/>
|
||||
</svg>
|
||||
Local Site Identity
|
||||
</h5>
|
||||
<p class="text-muted small mb-0">This site's configuration</p>
|
||||
</div>
|
||||
<div class="card-body px-4 pb-4">
|
||||
<form method="POST" action="{{ url_for('ui.update_local_site') }}" id="localSiteForm">
|
||||
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
|
||||
<div class="mb-3">
|
||||
<label for="site_id" class="form-label fw-medium">Site ID</label>
|
||||
<input type="text" class="form-control" id="site_id" name="site_id" required
|
||||
value="{{ local_site.site_id if local_site else config_site_id or '' }}"
|
||||
placeholder="us-west-1">
|
||||
<div class="form-text">Unique identifier for this site</div>
|
||||
</div>
|
||||
<div class="mb-3">
|
||||
<label for="endpoint" class="form-label fw-medium">Endpoint URL</label>
|
||||
<input type="url" class="form-control" id="endpoint" name="endpoint"
|
||||
value="{{ local_site.endpoint if local_site else config_site_endpoint or '' }}"
|
||||
placeholder="https://s3.us-west-1.example.com">
|
||||
<div class="form-text">Public URL for this site</div>
|
||||
</div>
|
||||
<div class="mb-3">
|
||||
<label for="region" class="form-label fw-medium">Region</label>
|
||||
<input type="text" class="form-control" id="region" name="region"
|
||||
value="{{ local_site.region if local_site else config_site_region }}">
|
||||
</div>
|
||||
<div class="row mb-3">
|
||||
<div class="col-6">
|
||||
<label for="priority" class="form-label fw-medium">Priority</label>
|
||||
<input type="number" class="form-control" id="priority" name="priority"
|
||||
value="{{ local_site.priority if local_site else 100 }}" min="0">
|
||||
<div class="form-text">Lower = preferred</div>
|
||||
</div>
|
||||
<div class="col-6">
|
||||
<label for="display_name" class="form-label fw-medium">Display Name</label>
|
||||
<input type="text" class="form-control" id="display_name" name="display_name"
|
||||
value="{{ local_site.display_name if local_site else '' }}"
|
||||
placeholder="US West Primary">
|
||||
</div>
|
||||
</div>
|
||||
<div class="d-grid">
|
||||
<button type="submit" class="btn btn-primary">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-1" viewBox="0 0 16 16">
|
||||
<path d="M10.97 4.97a.75.75 0 0 1 1.07 1.05l-3.99 4.99a.75.75 0 0 1-1.08.02L4.324 8.384a.75.75 0 1 1 1.06-1.06l2.094 2.093 3.473-4.425a.267.267 0 0 1 .02-.022z"/>
|
||||
</svg>
|
||||
Save Local Site
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card shadow-sm border-0" style="border-radius: 1rem;">
|
||||
<div class="card-header bg-transparent border-0 pt-3 pb-0 px-4">
|
||||
<button class="btn btn-link text-decoration-none p-0 w-100 d-flex align-items-center justify-content-between"
|
||||
type="button" data-bs-toggle="collapse" data-bs-target="#addPeerCollapse"
|
||||
aria-expanded="false" aria-controls="addPeerCollapse">
|
||||
<span class="d-flex align-items-center gap-2">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" class="text-primary" viewBox="0 0 16 16">
|
||||
<path fill-rule="evenodd" d="M8 2a.5.5 0 0 1 .5.5v5h5a.5.5 0 0 1 0 1h-5v5a.5.5 0 0 1-1 0v-5h-5a.5.5 0 0 1 0-1h5v-5A.5.5 0 0 1 8 2Z"/>
|
||||
</svg>
|
||||
<span class="fw-semibold h5 mb-0">Add Peer Site</span>
|
||||
</span>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="text-muted add-peer-chevron" viewBox="0 0 16 16">
|
||||
<path fill-rule="evenodd" d="M1.646 4.646a.5.5 0 0 1 .708 0L8 10.293l5.646-5.647a.5.5 0 0 1 .708.708l-6 6a.5.5 0 0 1-.708 0l-6-6a.5.5 0 0 1 0-.708z"/>
|
||||
</svg>
|
||||
</button>
|
||||
<p class="text-muted small mb-0 mt-1">Register a remote site</p>
|
||||
</div>
|
||||
<div class="collapse" id="addPeerCollapse">
|
||||
<div class="card-body px-4 pb-4">
|
||||
<form method="POST" action="{{ url_for('ui.add_peer_site') }}" id="addPeerForm">
|
||||
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
|
||||
<div class="mb-3">
|
||||
<label for="peer_site_id" class="form-label fw-medium">Site ID</label>
|
||||
<input type="text" class="form-control" id="peer_site_id" name="site_id" required placeholder="us-east-1">
|
||||
</div>
|
||||
<div class="mb-3">
|
||||
<label for="peer_endpoint" class="form-label fw-medium">Endpoint URL</label>
|
||||
<input type="url" class="form-control" id="peer_endpoint" name="endpoint" required placeholder="https://s3.us-east-1.example.com">
|
||||
</div>
|
||||
<div class="mb-3">
|
||||
<label for="peer_region" class="form-label fw-medium">Region</label>
|
||||
<input type="text" class="form-control" id="peer_region" name="region" value="us-east-1">
|
||||
</div>
|
||||
<div class="row mb-3">
|
||||
<div class="col-6">
|
||||
<label for="peer_priority" class="form-label fw-medium">Priority</label>
|
||||
<input type="number" class="form-control" id="peer_priority" name="priority" value="100" min="0">
|
||||
</div>
|
||||
<div class="col-6">
|
||||
<label for="peer_display_name" class="form-label fw-medium">Display Name</label>
|
||||
<input type="text" class="form-control" id="peer_display_name" name="display_name" placeholder="US East DR">
|
||||
</div>
|
||||
</div>
|
||||
<div class="mb-3">
|
||||
<label for="peer_connection_id" class="form-label fw-medium">Connection</label>
|
||||
<select class="form-select" id="peer_connection_id" name="connection_id">
|
||||
<option value="">No connection</option>
|
||||
{% for conn in connections %}
|
||||
<option value="{{ conn.id }}">{{ conn.name }} ({{ conn.endpoint_url }})</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
<div class="form-text">Link to a remote connection for health checks</div>
|
||||
</div>
|
||||
<div class="d-grid">
|
||||
<button type="submit" class="btn btn-primary">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-1" viewBox="0 0 16 16">
|
||||
<path fill-rule="evenodd" d="M8 2a.5.5 0 0 1 .5.5v5h5a.5.5 0 0 1 0 1h-5v5a.5.5 0 0 1-1 0v-5h-5a.5.5 0 0 1 0-1h5v-5A.5.5 0 0 1 8 2Z"/>
|
||||
</svg>
|
||||
Add Peer Site
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="col-lg-8 col-md-7">
|
||||
<div class="card shadow-sm border-0" style="border-radius: 1rem;">
|
||||
<div class="card-header bg-transparent border-0 pt-4 pb-0 px-4 d-flex justify-content-between align-items-start">
|
||||
<div>
|
||||
<h5 class="fw-semibold d-flex align-items-center gap-2 mb-1">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" class="text-muted" viewBox="0 0 16 16">
|
||||
<path fill-rule="evenodd" d="M6 3.5A1.5 1.5 0 0 1 7.5 2h1A1.5 1.5 0 0 1 10 3.5v1A1.5 1.5 0 0 1 8.5 6v1H14a.5.5 0 0 1 .5.5v1a.5.5 0 0 1-1 0V8h-5v.5a.5.5 0 0 1-1 0V8h-5v.5a.5.5 0 0 1-1 0v-1A.5.5 0 0 1 2 7h5.5V6A1.5 1.5 0 0 1 6 4.5v-1zM8.5 5a.5.5 0 0 0 .5-.5v-1a.5.5 0 0 0-.5-.5h-1a.5.5 0 0 0-.5.5v1a.5.5 0 0 0 .5.5h1zM0 11.5A1.5 1.5 0 0 1 1.5 10h1A1.5 1.5 0 0 1 4 11.5v1A1.5 1.5 0 0 1 2.5 14h-1A1.5 1.5 0 0 1 0 12.5v-1zm1.5-.5a.5.5 0 0 0-.5.5v1a.5.5 0 0 0 .5.5h1a.5.5 0 0 0 .5-.5v-1a.5.5 0 0 0-.5-.5h-1zm4.5.5A1.5 1.5 0 0 1 7.5 10h1a1.5 1.5 0 0 1 1.5 1.5v1A1.5 1.5 0 0 1 8.5 14h-1A1.5 1.5 0 0 1 6 12.5v-1zm1.5-.5a.5.5 0 0 0-.5.5v1a.5.5 0 0 0 .5.5h1a.5.5 0 0 0 .5-.5v-1a.5.5 0 0 0-.5-.5h-1zm4.5.5a1.5 1.5 0 0 1 1.5-1.5h1a1.5 1.5 0 0 1 1.5 1.5v1a1.5 1.5 0 0 1-1.5 1.5h-1a1.5 1.5 0 0 1-1.5-1.5v-1zm1.5-.5a.5.5 0 0 0-.5.5v1a.5.5 0 0 0 .5.5h1a.5.5 0 0 0 .5-.5v-1a.5.5 0 0 0-.5-.5h-1z"/>
|
||||
</svg>
|
||||
Peer Sites
|
||||
</h5>
|
||||
<p class="text-muted small mb-0">Known remote sites in the cluster</p>
|
||||
</div>
|
||||
{% if peers %}
|
||||
<button type="button" class="btn btn-outline-secondary btn-sm" id="btnCheckAllHealth" title="Check health of all peers">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-1" viewBox="0 0 16 16">
|
||||
<path d="M11.251.068a.5.5 0 0 1 .227.58L9.677 6.5H13a.5.5 0 0 1 .364.843l-8 8.5a.5.5 0 0 1-.842-.49L6.323 9.5H3a.5.5 0 0 1-.364-.843l8-8.5a.5.5 0 0 1 .615-.09z"/>
|
||||
</svg>
|
||||
Check All
|
||||
</button>
|
||||
{% endif %}
|
||||
</div>
|
||||
<div class="card-body px-4 pb-4">
|
||||
{% if peers %}
|
||||
<div class="table-responsive">
|
||||
<table class="table table-hover align-middle mb-0">
|
||||
<thead class="table-light">
|
||||
<tr>
|
||||
<th scope="col" style="width: 50px;">Health</th>
|
||||
<th scope="col">Site ID</th>
|
||||
<th scope="col">Endpoint</th>
|
||||
<th scope="col">Region</th>
|
||||
<th scope="col">Priority</th>
|
||||
<th scope="col">Sync Status</th>
|
||||
<th scope="col" class="text-end">Actions</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for item in peers_with_stats %}
|
||||
{% set peer = item.peer %}
|
||||
<tr data-site-id="{{ peer.site_id }}">
|
||||
<td class="text-center">
|
||||
<span class="peer-health-status" data-site-id="{{ peer.site_id }}"
|
||||
data-last-checked="{{ peer.last_health_check or '' }}"
|
||||
title="{% if peer.is_healthy == true %}Healthy{% elif peer.is_healthy == false %}Unhealthy{% else %}Not checked{% endif %}{% if peer.last_health_check %} (checked {{ peer.last_health_check }}){% endif %}"
|
||||
style="cursor: help;">
|
||||
{% if peer.is_healthy == true %}
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="text-success" viewBox="0 0 16 16">
|
||||
<path d="M16 8A8 8 0 1 1 0 8a8 8 0 0 1 16 0zm-3.97-3.03a.75.75 0 0 0-1.08.022L7.477 9.417 5.384 7.323a.75.75 0 0 0-1.06 1.06L6.97 11.03a.75.75 0 0 0 1.079-.02l3.992-4.99a.75.75 0 0 0-.01-1.05z"/>
|
||||
</svg>
|
||||
{% elif peer.is_healthy == false %}
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="text-danger" viewBox="0 0 16 16">
|
||||
<path d="M16 8A8 8 0 1 1 0 8a8 8 0 0 1 16 0zM5.354 4.646a.5.5 0 1 0-.708.708L7.293 8l-2.647 2.646a.5.5 0 0 0 .708.708L8 8.707l2.646 2.647a.5.5 0 0 0 .708-.708L8.707 8l2.647-2.646a.5.5 0 0 0-.708-.708L8 7.293 5.354 4.646z"/>
|
||||
</svg>
|
||||
{% else %}
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="text-muted" viewBox="0 0 16 16">
|
||||
<path d="M8 15A7 7 0 1 1 8 1a7 7 0 0 1 0 14zm0 1A8 8 0 1 0 8 0a8 8 0 0 0 0 16z"/>
|
||||
<path d="M5.255 5.786a.237.237 0 0 0 .241.247h.825c.138 0 .248-.113.266-.25.09-.656.54-1.134 1.342-1.134.686 0 1.314.343 1.314 1.168 0 .635-.374.927-.965 1.371-.673.489-1.206 1.06-1.168 1.987l.003.217a.25.25 0 0 0 .25.246h.811a.25.25 0 0 0 .25-.25v-.105c0-.718.273-.927 1.01-1.486.609-.463 1.244-.977 1.244-2.056 0-1.511-1.276-2.241-2.673-2.241-1.267 0-2.655.59-2.75 2.286zm1.557 5.763c0 .533.425.927 1.01.927.609 0 1.028-.394 1.028-.927 0-.552-.42-.94-1.029-.94-.584 0-1.009.388-1.009.94z"/>
|
||||
</svg>
|
||||
{% endif %}
|
||||
</span>
|
||||
</td>
|
||||
<td>
|
||||
<div class="d-flex align-items-center gap-2">
|
||||
<div class="peer-icon">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" viewBox="0 0 16 16">
|
||||
<path d="M0 8a8 8 0 1 1 16 0A8 8 0 0 1 0 8zm7.5-6.923c-.67.204-1.335.82-1.887 1.855A7.97 7.97 0 0 0 5.145 4H7.5V1.077zM4.09 4a9.267 9.267 0 0 1 .64-1.539 6.7 6.7 0 0 1 .597-.933A7.025 7.025 0 0 0 2.255 4H4.09zm-.582 3.5c.03-.877.138-1.718.312-2.5H1.674a6.958 6.958 0 0 0-.656 2.5h2.49zM4.847 5a12.5 12.5 0 0 0-.338 2.5H7.5V5H4.847zM8.5 5v2.5h2.99a12.495 12.495 0 0 0-.337-2.5H8.5zM4.51 8.5a12.5 12.5 0 0 0 .337 2.5H7.5V8.5H4.51zm3.99 0V11h2.653c.187-.765.306-1.608.338-2.5H8.5z"/>
|
||||
</svg>
|
||||
</div>
|
||||
<div>
|
||||
<span class="fw-medium">{{ peer.display_name or peer.site_id }}</span>
|
||||
{% if peer.display_name and peer.display_name != peer.site_id %}
|
||||
<br><small class="text-muted">{{ peer.site_id }}</small>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
</td>
|
||||
<td>
|
||||
<span class="endpoint-display text-muted small" data-full-url="{{ peer.endpoint }}" title="{{ peer.endpoint }}" style="cursor: pointer;">
|
||||
{% set parsed = peer.endpoint.split('//') %}
|
||||
{% if parsed|length > 1 %}{{ parsed[1].split('/')[0] }}{% else %}{{ peer.endpoint }}{% endif %}
|
||||
</span>
|
||||
<button type="button" class="btn btn-link btn-sm p-0 ms-1 btn-copy-endpoint" data-url="{{ peer.endpoint }}" title="Copy full URL">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="12" height="12" fill="currentColor" class="text-muted" viewBox="0 0 16 16">
|
||||
<path d="M4 1.5H3a2 2 0 0 0-2 2V14a2 2 0 0 0 2 2h10a2 2 0 0 0 2-2V3.5a2 2 0 0 0-2-2h-1v1h1a1 1 0 0 1 1 1V14a1 1 0 0 1-1 1H3a1 1 0 0 1-1-1V3.5a1 1 0 0 1 1-1h1v-1z"/>
|
||||
<path d="M9.5 1a.5.5 0 0 1 .5.5v1a.5.5 0 0 1-.5.5h-3a.5.5 0 0 1-.5-.5v-1a.5.5 0 0 1 .5-.5h3zm-3-1A1.5 1.5 0 0 0 5 1.5v1A1.5 1.5 0 0 0 6.5 4h3A1.5 1.5 0 0 0 11 2.5v-1A1.5 1.5 0 0 0 9.5 0h-3z"/>
|
||||
</svg>
|
||||
</button>
|
||||
</td>
|
||||
<td><span class="text-muted small">{{ peer.region }}</span></td>
|
||||
<td><span class="text-muted small">{{ peer.priority }}</span></td>
|
||||
<td class="sync-stats-cell" data-site-id="{{ peer.site_id }}">
|
||||
{% if item.has_connection %}
|
||||
<div class="d-flex align-items-center gap-2">
|
||||
<span class="badge bg-primary bg-opacity-10 text-primary">{{ item.buckets_syncing }} bucket{{ 's' if item.buckets_syncing != 1 else '' }}</span>
|
||||
{% if item.has_bidirectional %}
|
||||
<span class="bidir-status-icon" data-site-id="{{ peer.site_id }}" title="Bidirectional sync - click to verify">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="text-info" viewBox="0 0 16 16" style="cursor: pointer;">
|
||||
<path fill-rule="evenodd" d="M1 11.5a.5.5 0 0 0 .5.5h11.793l-3.147 3.146a.5.5 0 0 0 .708.708l4-4a.5.5 0 0 0 0-.708l-4-4a.5.5 0 0 0-.708.708L13.293 11H1.5a.5.5 0 0 0-.5.5zm14-7a.5.5 0 0 1-.5.5H2.707l3.147 3.146a.5.5 0 1 1-.708.708l-4-4a.5.5 0 0 1 0-.708l4-4a.5.5 0 1 1 .708.708L2.707 4H14.5a.5.5 0 0 1 .5.5z"/>
|
||||
</svg>
|
||||
</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
<div class="sync-stats-detail d-none mt-2 small" id="stats-{{ peer.site_id }}">
|
||||
<span class="spinner-border spinner-border-sm text-muted" style="width: 12px; height: 12px;"></span>
|
||||
</div>
|
||||
{% else %}
|
||||
<a href="#" class="text-muted small link-no-connection"
|
||||
data-site-id="{{ peer.site_id }}"
|
||||
title="Click to link a connection">Link a connection</a>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td class="text-end">
|
||||
<div class="d-flex align-items-center justify-content-end gap-1">
|
||||
<button type="button" class="btn btn-outline-secondary btn-sm"
|
||||
data-bs-toggle="modal"
|
||||
data-bs-target="#editPeerModal"
|
||||
data-site-id="{{ peer.site_id }}"
|
||||
data-endpoint="{{ peer.endpoint }}"
|
||||
data-region="{{ peer.region }}"
|
||||
data-priority="{{ peer.priority }}"
|
||||
data-display-name="{{ peer.display_name }}"
|
||||
data-connection-id="{{ peer.connection_id or '' }}"
|
||||
title="Edit peer">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" viewBox="0 0 16 16">
|
||||
<path d="M12.146.146a.5.5 0 0 1 .708 0l3 3a.5.5 0 0 1 0 .708l-10 10a.5.5 0 0 1-.168.11l-5 2a.5.5 0 0 1-.65-.65l2-5a.5.5 0 0 1 .11-.168l10-10zM11.207 2.5 13.5 4.793 14.793 3.5 12.5 1.207 11.207 2.5zm1.586 3L10.5 3.207 4 9.707V10h.5a.5.5 0 0 1 .5.5v.5h.5a.5.5 0 0 1 .5.5v.5h.293l6.5-6.5z"/>
|
||||
</svg>
|
||||
</button>
|
||||
<div class="dropdown peer-actions-dropdown">
|
||||
<button class="btn btn-outline-secondary btn-sm" type="button" data-bs-toggle="dropdown" aria-expanded="false" title="More actions">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" viewBox="0 0 16 16">
|
||||
<path d="M3 9.5a1.5 1.5 0 1 1 0-3 1.5 1.5 0 0 1 0 3zm5 0a1.5 1.5 0 1 1 0-3 1.5 1.5 0 0 1 0 3zm5 0a1.5 1.5 0 1 1 0-3 1.5 1.5 0 0 1 0 3z"/>
|
||||
</svg>
|
||||
</button>
|
||||
<ul class="dropdown-menu dropdown-menu-end">
|
||||
<li>
|
||||
<button type="button" class="dropdown-item btn-check-health" data-site-id="{{ peer.site_id }}">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-2 text-warning" viewBox="0 0 16 16">
|
||||
<path d="M11.251.068a.5.5 0 0 1 .227.58L9.677 6.5H13a.5.5 0 0 1 .364.843l-8 8.5a.5.5 0 0 1-.842-.49L6.323 9.5H3a.5.5 0 0 1-.364-.843l8-8.5a.5.5 0 0 1 .615-.09z"/>
|
||||
</svg>
|
||||
Check Health
|
||||
</button>
|
||||
</li>
|
||||
<li>
|
||||
<button type="button" class="dropdown-item btn-check-bidir {% if not item.has_connection %}disabled{% endif %}"
|
||||
data-site-id="{{ peer.site_id }}"
|
||||
data-display-name="{{ peer.display_name or peer.site_id }}">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-2 text-info" viewBox="0 0 16 16">
|
||||
<path fill-rule="evenodd" d="M1 11.5a.5.5 0 0 0 .5.5h11.793l-3.147 3.146a.5.5 0 0 0 .708.708l4-4a.5.5 0 0 0 0-.708l-4-4a.5.5 0 0 0-.708.708L13.293 11H1.5a.5.5 0 0 0-.5.5zm14-7a.5.5 0 0 1-.5.5H2.707l3.147 3.146a.5.5 0 1 1-.708.708l-4-4a.5.5 0 0 1 0-.708l4-4a.5.5 0 1 1 .708.708L2.707 4H14.5a.5.5 0 0 1 .5.5z"/>
|
||||
</svg>
|
||||
Bidirectional Status
|
||||
</button>
|
||||
</li>
|
||||
{% if item.has_connection and item.buckets_syncing > 0 %}
|
||||
<li>
|
||||
<button type="button" class="dropdown-item btn-load-stats" data-site-id="{{ peer.site_id }}">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-2 text-primary" viewBox="0 0 16 16">
|
||||
<path fill-rule="evenodd" d="M8 3a5 5 0 1 0 4.546 2.914.5.5 0 0 1 .908-.417A6 6 0 1 1 8 2v1z"/>
|
||||
<path d="M8 4.466V.534a.25.25 0 0 1 .41-.192l2.36 1.966c.12.1.12.284 0 .384L8.41 4.658A.25.25 0 0 1 8 4.466z"/>
|
||||
</svg>
|
||||
Load Sync Stats
|
||||
</button>
|
||||
</li>
|
||||
{% endif %}
|
||||
<li>
|
||||
<a href="{{ url_for('ui.replication_wizard', site_id=peer.site_id) }}"
|
||||
class="dropdown-item {% if not item.has_connection %}disabled{% endif %}">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-2 text-primary" viewBox="0 0 16 16">
|
||||
<path d="M11.534 7h3.932a.25.25 0 0 1 .192.41l-1.966 2.36a.25.25 0 0 1-.384 0l-1.966-2.36a.25.25 0 0 1 .192-.41zm-11 2h3.932a.25.25 0 0 0 .192-.41L2.692 6.23a.25.25 0 0 0-.384 0L.342 8.59A.25.25 0 0 0 .534 9z"/>
|
||||
<path fill-rule="evenodd" d="M8 3c-1.552 0-2.94.707-3.857 1.818a.5.5 0 1 1-.771-.636A6.002 6.002 0 0 1 13.917 7H12.9A5.002 5.002 0 0 0 8 3zM3.1 9a5.002 5.002 0 0 0 8.757 2.182.5.5 0 1 1 .771.636A6.002 6.002 0 0 1 2.083 9H3.1z"/>
|
||||
</svg>
|
||||
Replication Wizard
|
||||
</a>
|
||||
</li>
|
||||
<li><hr class="dropdown-divider"></li>
|
||||
<li>
|
||||
<button type="button" class="dropdown-item text-danger"
|
||||
data-bs-toggle="modal"
|
||||
data-bs-target="#deletePeerModal"
|
||||
data-site-id="{{ peer.site_id }}"
|
||||
data-display-name="{{ peer.display_name or peer.site_id }}">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-2" viewBox="0 0 16 16">
|
||||
<path d="M5.5 5.5A.5.5 0 0 1 6 6v6a.5.5 0 0 1-1 0V6a.5.5 0 0 1 .5-.5zm2.5 0a.5.5 0 0 1 .5.5v6a.5.5 0 0 1-1 0V6a.5.5 0 0 1 .5-.5zm3 .5a.5.5 0 0 0-1 0v6a.5.5 0 0 0 1 0V6z"/>
|
||||
<path fill-rule="evenodd" d="M14.5 3a1 1 0 0 1-1 1H13v9a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2V4h-.5a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1H6a1 1 0 0 1 1-1h2a1 1 0 0 1 1 1h3.5a1 1 0 0 1 1 1v1zM4.118 4 4 4.059V13a1 1 0 0 0 1 1h6a1 1 0 0 0 1-1V4.059L11.882 4H4.118zM2.5 3V2h11v1h-11z"/>
|
||||
</svg>
|
||||
Delete Peer
|
||||
</button>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{% else %}
|
||||
<div class="empty-state text-center py-5">
|
||||
<div class="empty-state-icon mx-auto mb-3">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="48" height="48" fill="currentColor" viewBox="0 0 16 16">
|
||||
<path d="M0 8a8 8 0 1 1 16 0A8 8 0 0 1 0 8zm7.5-6.923c-.67.204-1.335.82-1.887 1.855A7.97 7.97 0 0 0 5.145 4H7.5V1.077zM4.09 4a9.267 9.267 0 0 1 .64-1.539 6.7 6.7 0 0 1 .597-.933A7.025 7.025 0 0 0 2.255 4H4.09zm-.582 3.5c.03-.877.138-1.718.312-2.5H1.674a6.958 6.958 0 0 0-.656 2.5h2.49zM4.847 5a12.5 12.5 0 0 0-.338 2.5H7.5V5H4.847zM8.5 5v2.5h2.99a12.495 12.495 0 0 0-.337-2.5H8.5zM4.51 8.5a12.5 12.5 0 0 0 .337 2.5H7.5V8.5H4.51zm3.99 0V11h2.653c.187-.765.306-1.608.338-2.5H8.5zM5.145 12c.138.386.295.744.468 1.068.552 1.035 1.218 1.65 1.887 1.855V12H5.145zm.182 2.472a6.696 6.696 0 0 1-.597-.933A9.268 9.268 0 0 1 4.09 12H2.255a7.024 7.024 0 0 0 3.072 2.472z"/>
|
||||
</svg>
|
||||
</div>
|
||||
<h5 class="fw-semibold mb-2">No peer sites yet</h5>
|
||||
<p class="text-muted mb-0">Add peer sites to enable geo-distribution and site-to-site replication.</p>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="modal fade" id="editPeerModal" tabindex="-1" aria-hidden="true">
|
||||
<div class="modal-dialog modal-dialog-centered">
|
||||
<div class="modal-content">
|
||||
<div class="modal-header border-0 pb-0">
|
||||
<h5 class="modal-title fw-semibold">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" class="text-primary" viewBox="0 0 16 16">
|
||||
<path d="M12.146.146a.5.5 0 0 1 .708 0l3 3a.5.5 0 0 1 0 .708l-10 10a.5.5 0 0 1-.168.11l-5 2a.5.5 0 0 1-.65-.65l2-5a.5.5 0 0 1 .11-.168l10-10zM11.207 2.5 13.5 4.793 14.793 3.5 12.5 1.207 11.207 2.5zm1.586 3L10.5 3.207 4 9.707V10h.5a.5.5 0 0 1 .5.5v.5h.5a.5.5 0 0 1 .5.5v.5h.293l6.5-6.5zm-9.761 5.175-.106.106-1.528 3.821 3.821-1.528.106-.106A.5.5 0 0 1 5 12.5V12h-.5a.5.5 0 0 1-.5-.5V11h-.5a.5.5 0 0 1-.468-.325z"/>
|
||||
</svg>
|
||||
Edit Peer Site
|
||||
</h5>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
||||
</div>
|
||||
<form method="POST" id="editPeerForm">
|
||||
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
|
||||
<div class="modal-body">
|
||||
<div class="mb-3">
|
||||
<label class="form-label fw-medium">Site ID</label>
|
||||
<input type="text" class="form-control" id="edit_site_id" readonly>
|
||||
</div>
|
||||
<div class="mb-3">
|
||||
<label for="edit_endpoint" class="form-label fw-medium">Endpoint URL</label>
|
||||
<input type="url" class="form-control" id="edit_endpoint" name="endpoint" required>
|
||||
</div>
|
||||
<div class="mb-3">
|
||||
<label for="edit_region" class="form-label fw-medium">Region</label>
|
||||
<input type="text" class="form-control" id="edit_region" name="region" required>
|
||||
</div>
|
||||
<div class="row mb-3">
|
||||
<div class="col-6">
|
||||
<label for="edit_priority" class="form-label fw-medium">Priority</label>
|
||||
<input type="number" class="form-control" id="edit_priority" name="priority" min="0">
|
||||
</div>
|
||||
<div class="col-6">
|
||||
<label for="edit_display_name" class="form-label fw-medium">Display Name</label>
|
||||
<input type="text" class="form-control" id="edit_display_name" name="display_name">
|
||||
</div>
|
||||
</div>
|
||||
<div class="mb-3">
|
||||
<label for="edit_connection_id" class="form-label fw-medium">Connection</label>
|
||||
<select class="form-select" id="edit_connection_id" name="connection_id">
|
||||
<option value="">No connection</option>
|
||||
{% for conn in connections %}
|
||||
<option value="{{ conn.id }}">{{ conn.name }} ({{ conn.endpoint_url }})</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="btn btn-outline-secondary" data-bs-dismiss="modal">Cancel</button>
|
||||
<button type="submit" class="btn btn-primary">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-1" viewBox="0 0 16 16">
|
||||
<path d="M10.97 4.97a.75.75 0 0 1 1.07 1.05l-3.99 4.99a.75.75 0 0 1-1.08.02L4.324 8.384a.75.75 0 1 1 1.06-1.06l2.094 2.093 3.473-4.425a.267.267 0 0 1 .02-.022z"/>
|
||||
</svg>
|
||||
Save
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="modal fade" id="deletePeerModal" tabindex="-1" aria-hidden="true">
|
||||
<div class="modal-dialog modal-dialog-centered">
|
||||
<div class="modal-content">
|
||||
<div class="modal-header border-0 pb-0">
|
||||
<h5 class="modal-title fw-semibold">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" class="text-danger" viewBox="0 0 16 16">
|
||||
<path d="M5.5 5.5A.5.5 0 0 1 6 6v6a.5.5 0 0 1-1 0V6a.5.5 0 0 1 .5-.5zm2.5 0a.5.5 0 0 1 .5.5v6a.5.5 0 0 1-1 0V6a.5.5 0 0 1 .5-.5zm3 .5a.5.5 0 0 0-1 0v6a.5.5 0 0 0 1 0V6z"/>
|
||||
<path fill-rule="evenodd" d="M14.5 3a1 1 0 0 1-1 1H13v9a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2V4h-.5a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1H6a1 1 0 0 1 1-1h2a1 1 0 0 1 1 1h3.5a1 1 0 0 1 1 1v1zM4.118 4 4 4.059V13a1 1 0 0 0 1 1h6a1 1 0 0 0 1-1V4.059L11.882 4H4.118zM2.5 3V2h11v1h-11z"/>
|
||||
</svg>
|
||||
Delete Peer Site
|
||||
</h5>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
||||
</div>
|
||||
<div class="modal-body">
|
||||
<p>Are you sure you want to delete <strong id="deletePeerName"></strong>?</p>
|
||||
<div class="alert alert-warning d-flex align-items-start small" role="alert">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="flex-shrink-0 me-2 mt-0" viewBox="0 0 16 16">
|
||||
<path d="M8 16A8 8 0 1 0 8 0a8 8 0 0 0 0 16zm.93-9.412-1 4.705c-.07.34.029.533.304.533.194 0 .487-.07.686-.246l-.088.416c-.287.346-.92.598-1.465.598-.703 0-1.002-.422-.808-1.319l.738-3.468c.064-.293.006-.399-.287-.47l-.451-.081.082-.381 2.29-.287zM8 5.5a1 1 0 1 1 0-2 1 1 0 0 1 0 2z"/>
|
||||
</svg>
|
||||
<div>This will remove the peer from the site registry. Any site sync configurations may be affected.</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="btn btn-outline-secondary" data-bs-dismiss="modal">Cancel</button>
|
||||
<form method="POST" id="deletePeerForm">
|
||||
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
|
||||
<button type="submit" class="btn btn-danger">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-1" viewBox="0 0 16 16">
|
||||
<path d="M5.5 5.5A.5.5 0 0 1 6 6v6a.5.5 0 0 1-1 0V6a.5.5 0 0 1 .5-.5zm2.5 0a.5.5 0 0 1 .5.5v6a.5.5 0 0 1-1 0V6a.5.5 0 0 1 .5-.5zm3 .5a.5.5 0 0 0-1 0v6a.5.5 0 0 0 1 0V6z"/>
|
||||
<path fill-rule="evenodd" d="M14.5 3a1 1 0 0 1-1 1H13v9a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2V4h-.5a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1H6a1 1 0 0 1 1-1h2a1 1 0 0 1 1 1h3.5a1 1 0 0 1 1 1v1zM4.118 4 4 4.059V13a1 1 0 0 0 1 1h6a1 1 0 0 0 1-1V4.059L11.882 4H4.118zM2.5 3V2h11v1h-11z"/>
|
||||
</svg>
|
||||
Delete
|
||||
</button>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="modal fade" id="bidirStatusModal" tabindex="-1" aria-hidden="true">
|
||||
<div class="modal-dialog modal-dialog-centered modal-lg">
|
||||
<div class="modal-content">
|
||||
<div class="modal-header border-0 pb-0">
|
||||
<h5 class="modal-title fw-semibold">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" class="text-info me-2" viewBox="0 0 16 16">
|
||||
<path fill-rule="evenodd" d="M1 11.5a.5.5 0 0 0 .5.5h11.793l-3.147 3.146a.5.5 0 0 0 .708.708l4-4a.5.5 0 0 0 0-.708l-4-4a.5.5 0 0 0-.708.708L13.293 11H1.5a.5.5 0 0 0-.5.5zm14-7a.5.5 0 0 1-.5.5H2.707l3.147 3.146a.5.5 0 1 1-.708.708l-4-4a.5.5 0 0 1 0-.708l4-4a.5.5 0 1 1 .708.708L2.707 4H14.5a.5.5 0 0 1 .5.5z"/>
|
||||
</svg>
|
||||
Bidirectional Sync Status
|
||||
</h5>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
||||
</div>
|
||||
<div class="modal-body">
|
||||
<div id="bidirStatusContent">
|
||||
<div class="text-center py-4">
|
||||
<span class="spinner-border text-primary" role="status"></span>
|
||||
<p class="text-muted mt-2 mb-0">Checking configuration...</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="btn btn-outline-secondary" data-bs-dismiss="modal">Close</button>
|
||||
<a href="#" id="bidirWizardLink" class="btn btn-primary d-none">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-1" viewBox="0 0 16 16">
|
||||
<path d="M9.828.722a.5.5 0 0 1 .354.146l4.95 4.95a.5.5 0 0 1 0 .707c-.48.48-1.072.588-1.503.588-.177 0-.335-.018-.46-.039l-3.134 3.134a5.927 5.927 0 0 1 .16 1.013c.046.702-.032 1.687-.72 2.375a.5.5 0 0 1-.707 0l-2.829-2.828-3.182 3.182c-.195.195-1.219.902-1.414.707-.195-.195.512-1.22.707-1.414l3.182-3.182-2.828-2.829a.5.5 0 0 1 0-.707c.688-.688 1.673-.767 2.375-.72a5.922 5.922 0 0 1 1.013.16l3.134-3.133a2.772 2.772 0 0 1-.04-.461c0-.43.108-1.022.589-1.503a.5.5 0 0 1 .353-.146z"/>
|
||||
</svg>
|
||||
Run Setup Wizard
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
(function() {
|
||||
var escapeHtml = window.UICore ? window.UICore.escapeHtml : function(s) { return s; };
|
||||
|
||||
var editPeerModal = document.getElementById('editPeerModal');
|
||||
if (editPeerModal) {
|
||||
editPeerModal.addEventListener('show.bs.modal', function (event) {
|
||||
var button = event.relatedTarget;
|
||||
var siteId = button.getAttribute('data-site-id');
|
||||
document.getElementById('edit_site_id').value = siteId;
|
||||
document.getElementById('edit_endpoint').value = button.getAttribute('data-endpoint');
|
||||
document.getElementById('edit_region').value = button.getAttribute('data-region');
|
||||
document.getElementById('edit_priority').value = button.getAttribute('data-priority');
|
||||
document.getElementById('edit_display_name').value = button.getAttribute('data-display-name');
|
||||
document.getElementById('edit_connection_id').value = button.getAttribute('data-connection-id');
|
||||
document.getElementById('editPeerForm').action = '/ui/sites/peers/' + encodeURIComponent(siteId) + '/update';
|
||||
});
|
||||
}
|
||||
|
||||
document.querySelectorAll('.link-no-connection').forEach(function(link) {
|
||||
link.addEventListener('click', function(e) {
|
||||
e.preventDefault();
|
||||
var siteId = this.getAttribute('data-site-id');
|
||||
var row = this.closest('tr[data-site-id]');
|
||||
if (row) {
|
||||
var btn = row.querySelector('.btn[data-bs-target="#editPeerModal"]');
|
||||
if (btn) btn.click();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
var deletePeerModal = document.getElementById('deletePeerModal');
|
||||
if (deletePeerModal) {
|
||||
deletePeerModal.addEventListener('show.bs.modal', function (event) {
|
||||
var button = event.relatedTarget;
|
||||
var siteId = button.getAttribute('data-site-id');
|
||||
var displayName = button.getAttribute('data-display-name');
|
||||
document.getElementById('deletePeerName').textContent = displayName;
|
||||
document.getElementById('deletePeerForm').action = '/ui/sites/peers/' + encodeURIComponent(siteId) + '/delete';
|
||||
});
|
||||
}
|
||||
|
||||
function formatTimeAgo(date) {
|
||||
var seconds = Math.floor((new Date() - date) / 1000);
|
||||
if (seconds < 60) return 'just now';
|
||||
var minutes = Math.floor(seconds / 60);
|
||||
if (minutes < 60) return minutes + 'm ago';
|
||||
var hours = Math.floor(minutes / 60);
|
||||
if (hours < 24) return hours + 'h ago';
|
||||
return Math.floor(hours / 24) + 'd ago';
|
||||
}
|
||||
|
||||
function doHealthCheck(siteId) {
|
||||
var row = document.querySelector('tr[data-site-id="' + CSS.escape(siteId) + '"]');
|
||||
var statusSpan = row ? row.querySelector('.peer-health-status') : null;
|
||||
if (!statusSpan) return Promise.resolve();
|
||||
|
||||
statusSpan.innerHTML = '<span class="spinner-border spinner-border-sm text-muted" role="status" style="width: 14px; height: 14px;"></span>';
|
||||
|
||||
return fetch('/ui/sites/peers/' + encodeURIComponent(siteId) + '/health')
|
||||
.then(function(response) { return response.json(); })
|
||||
.then(function(data) {
|
||||
var now = new Date();
|
||||
statusSpan.setAttribute('data-last-checked', now.toISOString());
|
||||
if (data.is_healthy) {
|
||||
statusSpan.innerHTML = '<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="text-success" viewBox="0 0 16 16"><path d="M16 8A8 8 0 1 1 0 8a8 8 0 0 1 16 0zm-3.97-3.03a.75.75 0 0 0-1.08.022L7.477 9.417 5.384 7.323a.75.75 0 0 0-1.06 1.06L6.97 11.03a.75.75 0 0 0 1.079-.02l3.992-4.99a.75.75 0 0 0-.01-1.05z"/></svg>';
|
||||
statusSpan.title = 'Healthy (checked ' + formatTimeAgo(now) + ')';
|
||||
return { siteId: siteId, healthy: true };
|
||||
} else {
|
||||
statusSpan.innerHTML = '<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="text-danger" viewBox="0 0 16 16"><path d="M16 8A8 8 0 1 1 0 8a8 8 0 0 1 16 0zM5.354 4.646a.5.5 0 1 0-.708.708L7.293 8l-2.647 2.646a.5.5 0 0 0 .708.708L8 8.707l2.646 2.647a.5.5 0 0 0 .708-.708L8.707 8l2.647-2.646a.5.5 0 0 0-.708-.708L8 7.293 5.354 4.646z"/></svg>';
|
||||
statusSpan.title = 'Unhealthy' + (data.error ? ': ' + data.error : '') + ' (checked ' + formatTimeAgo(now) + ')';
|
||||
return { siteId: siteId, healthy: false, error: data.error };
|
||||
}
|
||||
})
|
||||
.catch(function(err) {
|
||||
statusSpan.innerHTML = '<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="text-muted" viewBox="0 0 16 16"><path d="M8 15A7 7 0 1 1 8 1a7 7 0 0 1 0 14zm0 1A8 8 0 1 0 8 0a8 8 0 0 0 0 16z"/><path d="M5.255 5.786a.237.237 0 0 0 .241.247h.825c.138 0 .248-.113.266-.25.09-.656.54-1.134 1.342-1.134.686 0 1.314.343 1.314 1.168 0 .635-.374.927-.965 1.371-.673.489-1.206 1.06-1.168 1.987l.003.217a.25.25 0 0 0 .25.246h.811a.25.25 0 0 0 .25-.25v-.105c0-.718.273-.927 1.01-1.486.609-.463 1.244-.977 1.244-2.056 0-1.511-1.276-2.241-2.673-2.241-1.267 0-2.655.59-2.75 2.286zm1.557 5.763c0 .533.425.927 1.01.927.609 0 1.028-.394 1.028-.927 0-.552-.42-.94-1.029-.94-.584 0-1.009.388-1.009.94z"/></svg>';
|
||||
statusSpan.title = 'Check failed';
|
||||
return { siteId: siteId, healthy: null };
|
||||
});
|
||||
}
|
||||
|
||||
document.querySelectorAll('.btn-check-health').forEach(function(btn) {
|
||||
btn.addEventListener('click', function() {
|
||||
var siteId = this.getAttribute('data-site-id');
|
||||
doHealthCheck(siteId).then(function(result) {
|
||||
if (!result) return;
|
||||
if (result.healthy === true) {
|
||||
if (window.showToast) window.showToast('Peer site is healthy', 'Health Check', 'success');
|
||||
} else if (result.healthy === false) {
|
||||
if (window.showToast) window.showToast(result.error || 'Peer site is unhealthy', 'Health Check', 'error');
|
||||
} else {
|
||||
if (window.showToast) window.showToast('Failed to check health', 'Health Check', 'error');
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
var checkAllBtn = document.getElementById('btnCheckAllHealth');
|
||||
if (checkAllBtn) {
|
||||
checkAllBtn.addEventListener('click', function() {
|
||||
var btn = this;
|
||||
var originalHtml = btn.innerHTML;
|
||||
btn.disabled = true;
|
||||
btn.innerHTML = '<span class="spinner-border spinner-border-sm me-1"></span>Checking...';
|
||||
|
||||
var siteIds = [];
|
||||
document.querySelectorAll('.peer-health-status').forEach(function(el) {
|
||||
siteIds.push(el.getAttribute('data-site-id'));
|
||||
});
|
||||
|
||||
var promises = siteIds.map(function(id) { return doHealthCheck(id); });
|
||||
Promise.all(promises).then(function(results) {
|
||||
var healthy = results.filter(function(r) { return r && r.healthy === true; }).length;
|
||||
var unhealthy = results.filter(function(r) { return r && r.healthy === false; }).length;
|
||||
var failed = results.filter(function(r) { return r && r.healthy === null; }).length;
|
||||
|
||||
var msg = healthy + ' healthy';
|
||||
if (unhealthy > 0) msg += ', ' + unhealthy + ' unhealthy';
|
||||
if (failed > 0) msg += ', ' + failed + ' failed';
|
||||
if (window.showToast) window.showToast(msg, 'Health Check', unhealthy > 0 ? 'warning' : 'success');
|
||||
|
||||
btn.disabled = false;
|
||||
btn.innerHTML = originalHtml;
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
document.querySelectorAll('.btn-load-stats').forEach(function(btn) {
|
||||
btn.addEventListener('click', function() {
|
||||
var siteId = this.getAttribute('data-site-id');
|
||||
var detailDiv = document.getElementById('stats-' + siteId);
|
||||
if (!detailDiv) return;
|
||||
|
||||
detailDiv.classList.remove('d-none');
|
||||
detailDiv.innerHTML = '<span class="spinner-border spinner-border-sm text-muted" style="width: 12px; height: 12px;"></span> Loading...';
|
||||
|
||||
fetch('/ui/sites/peers/' + encodeURIComponent(siteId) + '/sync-stats')
|
||||
.then(function(response) { return response.json(); })
|
||||
.then(function(data) {
|
||||
if (data.error) {
|
||||
detailDiv.innerHTML = '<span class="text-danger">' + escapeHtml(data.error) + '</span>';
|
||||
} else {
|
||||
var lastSync = data.last_sync_at
|
||||
? new Date(data.last_sync_at * 1000).toLocaleString()
|
||||
: 'Never';
|
||||
detailDiv.innerHTML =
|
||||
'<div class="d-flex flex-wrap gap-2 mb-1">' +
|
||||
'<span class="text-success"><strong>' + escapeHtml(String(data.objects_synced)) + '</strong> synced</span>' +
|
||||
'<span class="text-warning"><strong>' + escapeHtml(String(data.objects_pending)) + '</strong> pending</span>' +
|
||||
'<span class="text-danger"><strong>' + escapeHtml(String(data.objects_failed)) + '</strong> failed</span>' +
|
||||
'</div>' +
|
||||
'<div class="text-muted" style="font-size: 0.75rem;">Last sync: ' + escapeHtml(lastSync) + '</div>';
|
||||
}
|
||||
})
|
||||
.catch(function() {
|
||||
detailDiv.innerHTML = '<span class="text-danger">Failed to load stats</span>';
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
document.querySelectorAll('.bidir-status-icon').forEach(function(icon) {
|
||||
icon.addEventListener('click', function() {
|
||||
var siteId = this.getAttribute('data-site-id');
|
||||
var row = this.closest('tr[data-site-id]');
|
||||
var btn = row ? row.querySelector('.btn-check-bidir') : null;
|
||||
if (btn) btn.click();
|
||||
});
|
||||
});
|
||||
|
||||
document.querySelectorAll('.btn-check-bidir').forEach(function(btn) {
|
||||
btn.addEventListener('click', function() {
|
||||
var siteId = this.getAttribute('data-site-id');
|
||||
var displayName = this.getAttribute('data-display-name');
|
||||
var modal = new bootstrap.Modal(document.getElementById('bidirStatusModal'));
|
||||
var contentDiv = document.getElementById('bidirStatusContent');
|
||||
var wizardLink = document.getElementById('bidirWizardLink');
|
||||
|
||||
contentDiv.innerHTML =
|
||||
'<div class="text-center py-4">' +
|
||||
'<span class="spinner-border text-primary" role="status"></span>' +
|
||||
'<p class="text-muted mt-2 mb-0">Checking bidirectional configuration with ' + escapeHtml(displayName) + '...</p>' +
|
||||
'</div>';
|
||||
wizardLink.classList.add('d-none');
|
||||
modal.show();
|
||||
|
||||
fetch('/ui/sites/peers/' + encodeURIComponent(siteId) + '/bidirectional-status')
|
||||
.then(function(response) { return response.json(); })
|
||||
.then(function(data) {
|
||||
var html = '';
|
||||
|
||||
if (data.is_fully_configured) {
|
||||
html += '<div class="alert alert-success d-flex align-items-center mb-4" role="alert">' +
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="currentColor" class="flex-shrink-0 me-2" viewBox="0 0 16 16">' +
|
||||
'<path d="M16 8A8 8 0 1 1 0 8a8 8 0 0 1 16 0zm-3.97-3.03a.75.75 0 0 0-1.08.022L7.477 9.417 5.384 7.323a.75.75 0 0 0-1.06 1.06L6.97 11.03a.75.75 0 0 0 1.079-.02l3.992-4.99a.75.75 0 0 0-.01-1.05z"/>' +
|
||||
'</svg>' +
|
||||
'<div><strong>Bidirectional sync is fully configured!</strong><br><small>Both sites are set up to sync data in both directions.</small></div>' +
|
||||
'</div>';
|
||||
} else if (data.issues && data.issues.length > 0) {
|
||||
var errors = data.issues.filter(function(i) { return i.severity === 'error'; });
|
||||
var warnings = data.issues.filter(function(i) { return i.severity === 'warning'; });
|
||||
|
||||
if (errors.length > 0) {
|
||||
html += '<div class="alert alert-danger mb-3" role="alert">' +
|
||||
'<h6 class="alert-heading fw-bold mb-2">' +
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="me-1" viewBox="0 0 16 16"><path d="M16 8A8 8 0 1 1 0 8a8 8 0 0 1 16 0zM5.354 4.646a.5.5 0 1 0-.708.708L7.293 8l-2.647 2.646a.5.5 0 0 0 .708.708L8 8.707l2.646 2.647a.5.5 0 0 0 .708-.708L8.707 8l2.647-2.646a.5.5 0 0 0-.708-.708L8 7.293 5.354 4.646z"/></svg>' +
|
||||
' Configuration Errors</h6><ul class="mb-0 ps-3">';
|
||||
errors.forEach(function(issue) {
|
||||
html += '<li><strong>' + escapeHtml(issue.code) + ':</strong> ' + escapeHtml(issue.message) + '</li>';
|
||||
});
|
||||
html += '</ul></div>';
|
||||
}
|
||||
|
||||
if (warnings.length > 0) {
|
||||
html += '<div class="alert alert-warning mb-3" role="alert">' +
|
||||
'<h6 class="alert-heading fw-bold mb-2">' +
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="me-1" viewBox="0 0 16 16"><path d="M8.982 1.566a1.13 1.13 0 0 0-1.96 0L.165 13.233c-.457.778.091 1.767.98 1.767h13.713c.889 0 1.438-.99.98-1.767L8.982 1.566zM8 5c.535 0 .954.462.9.995l-.35 3.507a.552.552 0 0 1-1.1 0L7.1 5.995A.905.905 0 0 1 8 5zm.002 6a1 1 0 1 1 0 2 1 1 0 0 1 0-2z"/></svg>' +
|
||||
' Warnings</h6><ul class="mb-0 ps-3">';
|
||||
warnings.forEach(function(issue) {
|
||||
html += '<li><strong>' + escapeHtml(issue.code) + ':</strong> ' + escapeHtml(issue.message) + '</li>';
|
||||
});
|
||||
html += '</ul></div>';
|
||||
}
|
||||
}
|
||||
|
||||
html += '<div class="row g-3">';
|
||||
html += '<div class="col-md-6"><div class="card h-100"><div class="card-header bg-light py-2"><strong>This Site (Local)</strong></div>' +
|
||||
'<div class="card-body small">' +
|
||||
'<p class="mb-1"><strong>Site ID:</strong> ' + (data.local_site_id ? escapeHtml(data.local_site_id) : '<span class="text-danger">Not configured</span>') + '</p>' +
|
||||
'<p class="mb-1"><strong>Endpoint:</strong> ' + (data.local_endpoint ? escapeHtml(data.local_endpoint) : '<span class="text-danger">Not configured</span>') + '</p>' +
|
||||
'<p class="mb-1"><strong>Site Sync Worker:</strong> ' + (data.local_site_sync_enabled ? '<span class="text-success">Enabled</span>' : '<span class="text-warning">Disabled</span>') + '</p>' +
|
||||
'<p class="mb-0"><strong>Bidirectional Rules:</strong> ' + (data.local_bidirectional_rules ? data.local_bidirectional_rules.length : 0) + '</p>' +
|
||||
'</div></div></div>';
|
||||
|
||||
if (data.remote_status) {
|
||||
var rs = data.remote_status;
|
||||
html += '<div class="col-md-6"><div class="card h-100"><div class="card-header bg-light py-2"><strong>Remote Site (' + escapeHtml(displayName) + ')</strong></div>' +
|
||||
'<div class="card-body small">';
|
||||
if (rs.admin_access_denied) {
|
||||
html += '<p class="text-warning mb-0">Admin access denied - cannot verify remote configuration</p>';
|
||||
} else if (rs.reachable === false) {
|
||||
html += '<p class="text-danger mb-0">Could not reach remote admin API</p>';
|
||||
} else {
|
||||
html += '<p class="mb-1"><strong>Has Peer Entry for Us:</strong> ' + (rs.has_peer_for_us ? '<span class="text-success">Yes</span>' : '<span class="text-danger">No</span>') + '</p>' +
|
||||
'<p class="mb-1"><strong>Connection Configured:</strong> ' + (rs.peer_connection_configured ? '<span class="text-success">Yes</span>' : '<span class="text-danger">No</span>') + '</p>';
|
||||
}
|
||||
html += '</div></div></div>';
|
||||
} else {
|
||||
html += '<div class="col-md-6"><div class="card h-100"><div class="card-header bg-light py-2"><strong>Remote Site (' + escapeHtml(displayName) + ')</strong></div>' +
|
||||
'<div class="card-body small"><p class="text-muted mb-0">Could not check remote status</p></div></div></div>';
|
||||
}
|
||||
html += '</div>';
|
||||
|
||||
if (data.local_bidirectional_rules && data.local_bidirectional_rules.length > 0) {
|
||||
html += '<div class="mt-3"><h6 class="fw-semibold">Local Bidirectional Rules</h6>' +
|
||||
'<table class="table table-sm table-bordered mb-0"><thead class="table-light"><tr><th>Source Bucket</th><th>Target Bucket</th><th>Status</th></tr></thead><tbody>';
|
||||
data.local_bidirectional_rules.forEach(function(rule) {
|
||||
html += '<tr><td>' + escapeHtml(rule.bucket_name) + '</td><td>' + escapeHtml(rule.target_bucket) + '</td>' +
|
||||
'<td>' + (rule.enabled ? '<span class="badge bg-success">Enabled</span>' : '<span class="badge bg-secondary">Disabled</span>') + '</td></tr>';
|
||||
});
|
||||
html += '</tbody></table></div>';
|
||||
}
|
||||
|
||||
if (!data.is_fully_configured) {
|
||||
html += '<div class="alert alert-info mt-3 mb-0" role="alert">' +
|
||||
'<h6 class="alert-heading fw-bold">How to Fix</h6>' +
|
||||
'<ol class="mb-0 ps-3">' +
|
||||
'<li>Ensure this site has a Site ID and Endpoint URL configured</li>' +
|
||||
'<li>On the remote site, register this site as a peer with a connection</li>' +
|
||||
'<li>Create bidirectional replication rules on both sites</li>' +
|
||||
'<li>Enable SITE_SYNC_ENABLED=true on both sites</li>' +
|
||||
'</ol></div>';
|
||||
var blockingErrors = ['NO_CONNECTION', 'CONNECTION_NOT_FOUND', 'REMOTE_UNREACHABLE', 'ENDPOINT_NOT_ALLOWED'];
|
||||
var hasBlockingError = data.issues && data.issues.some(function(i) { return blockingErrors.indexOf(i.code) !== -1; });
|
||||
if (!hasBlockingError) {
|
||||
wizardLink.href = '/ui/sites/peers/' + encodeURIComponent(siteId) + '/replication-wizard';
|
||||
wizardLink.classList.remove('d-none');
|
||||
}
|
||||
}
|
||||
|
||||
contentDiv.innerHTML = html;
|
||||
})
|
||||
.catch(function(err) {
|
||||
contentDiv.innerHTML = '<div class="alert alert-danger" role="alert"><strong>Error:</strong> Failed to check bidirectional status. ' + escapeHtml(err.message || '') + '</div>';
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
document.querySelectorAll('.btn-copy-endpoint').forEach(function(btn) {
|
||||
btn.addEventListener('click', function(e) {
|
||||
e.stopPropagation();
|
||||
var url = this.getAttribute('data-url');
|
||||
if (window.UICore && window.UICore.copyToClipboard) {
|
||||
window.UICore.copyToClipboard(url).then(function(ok) {
|
||||
if (ok && window.showToast) window.showToast('Endpoint URL copied', 'Copied', 'success');
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
var localSiteForm = document.getElementById('localSiteForm');
|
||||
if (localSiteForm) {
|
||||
localSiteForm.addEventListener('submit', function(e) {
|
||||
e.preventDefault();
|
||||
window.UICore.submitFormAjax(this, {
|
||||
successMessage: 'Local site configuration updated',
|
||||
onSuccess: function() {
|
||||
setTimeout(function() { location.reload(); }, 800);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
var addPeerForm = document.getElementById('addPeerForm');
|
||||
if (addPeerForm) {
|
||||
addPeerForm.addEventListener('submit', function(e) {
|
||||
e.preventDefault();
|
||||
window.UICore.submitFormAjax(this, {
|
||||
successMessage: 'Peer site added',
|
||||
onSuccess: function(data) {
|
||||
if (data.redirect) {
|
||||
setTimeout(function() { window.location.href = data.redirect; }, 800);
|
||||
} else {
|
||||
setTimeout(function() { location.reload(); }, 800);
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
var editPeerForm = document.getElementById('editPeerForm');
|
||||
if (editPeerForm) {
|
||||
editPeerForm.addEventListener('submit', function(e) {
|
||||
e.preventDefault();
|
||||
var modal = bootstrap.Modal.getInstance(document.getElementById('editPeerModal'));
|
||||
window.UICore.submitFormAjax(this, {
|
||||
successMessage: 'Peer site updated',
|
||||
onSuccess: function() {
|
||||
if (modal) modal.hide();
|
||||
setTimeout(function() { location.reload(); }, 800);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
var deletePeerForm = document.getElementById('deletePeerForm');
|
||||
if (deletePeerForm) {
|
||||
deletePeerForm.addEventListener('submit', function(e) {
|
||||
e.preventDefault();
|
||||
var modal = bootstrap.Modal.getInstance(document.getElementById('deletePeerModal'));
|
||||
window.UICore.submitFormAjax(this, {
|
||||
successMessage: 'Peer site deleted',
|
||||
onSuccess: function() {
|
||||
if (modal) modal.hide();
|
||||
setTimeout(function() { location.reload(); }, 800);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
document.querySelectorAll('.peer-actions-dropdown').forEach(function(dd) {
|
||||
dd.addEventListener('shown.bs.dropdown', function() {
|
||||
var toggle = dd.querySelector('[data-bs-toggle="dropdown"]');
|
||||
var menu = dd.querySelector('.dropdown-menu');
|
||||
if (!toggle || !menu) return;
|
||||
var rect = toggle.getBoundingClientRect();
|
||||
menu.style.top = rect.bottom + 'px';
|
||||
menu.style.left = (rect.right - menu.offsetWidth) + 'px';
|
||||
});
|
||||
});
|
||||
})();
|
||||
</script>
|
||||
|
||||
<style>
|
||||
.add-peer-chevron {
|
||||
transition: transform 0.2s ease;
|
||||
}
|
||||
[aria-expanded="true"] .add-peer-chevron {
|
||||
transform: rotate(180deg);
|
||||
}
|
||||
.endpoint-display:hover {
|
||||
text-decoration: underline;
|
||||
}
|
||||
.peer-actions-dropdown .dropdown-menu {
|
||||
position: fixed !important;
|
||||
inset: auto !important;
|
||||
transform: none !important;
|
||||
}
|
||||
</style>
|
||||
{% endblock %}
|
||||
367
templates/website_domains.html
Normal file
367
templates/website_domains.html
Normal file
@@ -0,0 +1,367 @@
|
||||
{% extends "base.html" %}
|
||||
|
||||
{% block title %}Website Domains - MyFSIO Console{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<div class="page-header d-flex justify-content-between align-items-center mb-4">
|
||||
<div>
|
||||
<p class="text-uppercase text-muted small mb-1">Website Hosting</p>
|
||||
<h1 class="h3 mb-1 d-flex align-items-center gap-2">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="28" height="28" fill="currentColor" class="text-primary" viewBox="0 0 16 16">
|
||||
<path d="M0 8a8 8 0 1 1 16 0A8 8 0 0 1 0 8zm7.5-6.923c-.67.204-1.335.82-1.887 1.855A7.97 7.97 0 0 0 5.145 4H7.5V1.077zM4.09 4a9.267 9.267 0 0 1 .64-1.539 6.7 6.7 0 0 1 .597-.933A7.025 7.025 0 0 0 2.255 4H4.09zm-.582 3.5c.03-.877.138-1.718.312-2.5H1.674a6.958 6.958 0 0 0-.656 2.5h2.49zM4.847 5a12.5 12.5 0 0 0-.338 2.5H7.5V5H4.847zM8.5 5v2.5h2.99a12.495 12.495 0 0 0-.337-2.5H8.5zM4.51 8.5a12.5 12.5 0 0 0 .337 2.5H7.5V8.5H4.51zm3.99 0V11h2.653c.187-.765.306-1.608.338-2.5H8.5zM5.145 12c.138.386.295.744.468 1.068.552 1.035 1.218 1.65 1.887 1.855V12H5.145zm.182 2.472a6.696 6.696 0 0 1-.597-.933A9.268 9.268 0 0 1 4.09 12H2.255a7.024 7.024 0 0 0 3.072 2.472zM3.82 11a13.652 13.652 0 0 1-.312-2.5h-2.49c.062.89.291 1.733.656 2.5H3.82zm6.853 3.472A7.024 7.024 0 0 0 13.745 12H11.91a9.27 9.27 0 0 1-.64 1.539 6.688 6.688 0 0 1-.597.933zM8.5 12v2.923c.67-.204 1.335-.82 1.887-1.855.173-.324.33-.682.468-1.068H8.5zm3.68-1h2.146c.365-.767.594-1.61.656-2.5h-2.49a13.65 13.65 0 0 1-.312 2.5zm2.802-3.5a6.959 6.959 0 0 0-.656-2.5H12.18c.174.782.282 1.623.312 2.5h2.49zM11.27 2.461c.247.464.462.98.64 1.539h1.835a7.024 7.024 0 0 0-3.072-2.472c.218.284.418.598.597.933zM10.855 4a7.966 7.966 0 0 0-.468-1.068C9.835 1.897 9.17 1.282 8.5 1.077V4h2.355z"/>
|
||||
</svg>
|
||||
Domain Mappings
|
||||
</h1>
|
||||
<p class="text-muted mb-0 mt-1">Map custom domains to buckets for static website hosting.</p>
|
||||
</div>
|
||||
<div class="d-none d-md-block">
|
||||
<span class="badge bg-primary bg-opacity-10 text-primary fs-6 px-3 py-2">
|
||||
{{ mappings|length }} mapping{{ 's' if mappings|length != 1 else '' }}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="row g-4">
|
||||
<div class="col-lg-4 col-md-5">
|
||||
<div class="card shadow-sm border-0" style="border-radius: 1rem;">
|
||||
<div class="card-header bg-transparent border-0 pt-4 pb-0 px-4">
|
||||
<h5 class="fw-semibold d-flex align-items-center gap-2 mb-1">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" class="text-primary" viewBox="0 0 16 16">
|
||||
<path fill-rule="evenodd" d="M8 2a.5.5 0 0 1 .5.5v5h5a.5.5 0 0 1 0 1h-5v5a.5.5 0 0 1-1 0v-5h-5a.5.5 0 0 1 0-1h5v-5A.5.5 0 0 1 8 2Z"/>
|
||||
</svg>
|
||||
Add Domain Mapping
|
||||
</h5>
|
||||
<p class="text-muted small mb-0">Point a custom domain to a bucket</p>
|
||||
</div>
|
||||
<div class="card-body px-4 pb-4">
|
||||
<form method="POST" action="{{ url_for('ui.create_website_domain') }}" id="createDomainForm">
|
||||
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
|
||||
<div class="mb-3">
|
||||
<label for="domain" class="form-label fw-medium">Domain</label>
|
||||
<input type="text" class="form-control" id="domain" name="domain" required
|
||||
placeholder="www.example.com"
|
||||
pattern="^[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?(\.[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?)*$"
|
||||
title="Enter a valid hostname (e.g. www.example.com). Do not include http:// or trailing slashes.">
|
||||
<div class="form-text">Hostname only — no <code>http://</code> prefix or trailing slash.</div>
|
||||
<div class="invalid-feedback">Enter a valid hostname like www.example.com</div>
|
||||
</div>
|
||||
<div id="domainPreview" class="alert alert-light border small py-2 px-3 mb-3 d-none">
|
||||
<span class="text-muted">Will be accessible at:</span>
|
||||
<code id="domainPreviewUrl" class="ms-1"></code>
|
||||
</div>
|
||||
<div class="mb-3">
|
||||
<label for="bucket" class="form-label fw-medium">Bucket</label>
|
||||
{% if buckets %}
|
||||
<select class="form-select" id="bucket" name="bucket" required>
|
||||
<option value="" selected disabled>Select a bucket</option>
|
||||
{% for b in buckets %}
|
||||
<option value="{{ b }}">{{ b }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
{% else %}
|
||||
<input type="text" class="form-control" id="bucket" name="bucket" required placeholder="my-site-bucket">
|
||||
{% endif %}
|
||||
<div class="form-text">The bucket must have website hosting enabled.</div>
|
||||
</div>
|
||||
<div class="d-grid">
|
||||
<button type="submit" class="btn btn-primary" id="addMappingBtn">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-1" viewBox="0 0 16 16">
|
||||
<path fill-rule="evenodd" d="M8 2a.5.5 0 0 1 .5.5v5h5a.5.5 0 0 1 0 1h-5v5a.5.5 0 0 1-1 0v-5h-5a.5.5 0 0 1 0-1h5v-5A.5.5 0 0 1 8 2Z"/>
|
||||
</svg>
|
||||
Add Mapping
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card shadow-sm border-0 mt-4" style="border-radius: 1rem;">
|
||||
<div class="card-body px-4 py-3">
|
||||
<h6 class="fw-semibold mb-2 d-flex align-items-center gap-2">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="text-muted" viewBox="0 0 16 16">
|
||||
<path d="M8 16A8 8 0 1 0 8 0a8 8 0 0 0 0 16zm.93-9.412-1 4.705c-.07.34.029.533.304.533.194 0 .487-.07.686-.246l-.088.416c-.287.346-.92.598-1.465.598-.703 0-1.002-.422-.808-1.319l.738-3.468c.064-.293.006-.399-.287-.47l-.451-.081.082-.381 2.29-.287zM8 5.5a1 1 0 1 1 0-2 1 1 0 0 1 0 2z"/>
|
||||
</svg>
|
||||
How it works
|
||||
</h6>
|
||||
<ol class="small text-muted mb-0 ps-3">
|
||||
<li class="mb-1">Enable website hosting on a bucket (Properties tab)</li>
|
||||
<li class="mb-1">Create a domain mapping here</li>
|
||||
<li>Point your DNS (A/CNAME) to this server</li>
|
||||
</ol>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="col-lg-8 col-md-7">
|
||||
<div class="card shadow-sm border-0" style="border-radius: 1rem;">
|
||||
<div class="card-header bg-transparent border-0 pt-4 pb-0 px-4">
|
||||
<div class="d-flex justify-content-between align-items-center mb-1">
|
||||
<h5 class="fw-semibold d-flex align-items-center gap-2 mb-0">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" class="text-muted" viewBox="0 0 16 16">
|
||||
<path d="M4.715 6.542 3.343 7.914a3 3 0 1 0 4.243 4.243l1.828-1.829A3 3 0 0 0 8.586 5.5L8 6.086a1.002 1.002 0 0 0-.154.199 2 2 0 0 1 .861 3.337L6.88 11.45a2 2 0 1 1-2.83-2.83l.793-.792a4.018 4.018 0 0 1-.128-1.287z"/>
|
||||
<path d="M6.586 4.672A3 3 0 0 0 7.414 9.5l.775-.776a2 2 0 0 1-.896-3.346L9.12 3.55a2 2 0 1 1 2.83 2.83l-.793.792c.112.42.155.855.128 1.287l1.372-1.372a3 3 0 1 0-4.243-4.243L6.586 4.672z"/>
|
||||
</svg>
|
||||
Active Mappings
|
||||
</h5>
|
||||
</div>
|
||||
<p class="text-muted small mb-0">Domains currently serving website content</p>
|
||||
{% if mappings|length > 3 %}
|
||||
<div class="mt-3">
|
||||
<div class="search-input-wrapper">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="search-icon" viewBox="0 0 16 16">
|
||||
<path d="M11.742 10.344a6.5 6.5 0 1 0-1.397 1.398h-.001c.03.04.062.078.098.115l3.85 3.85a1 1 0 0 0 1.415-1.414l-3.85-3.85a1.007 1.007 0 0 0-.115-.1zM12 6.5a5.5 5.5 0 1 1-11 0 5.5 5.5 0 0 1 11 0z"/>
|
||||
</svg>
|
||||
<input type="text" class="form-control" id="domainSearch" placeholder="Filter by domain or bucket..." autocomplete="off" />
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
<div class="card-body px-4 pb-4">
|
||||
{% if mappings %}
|
||||
<div class="table-responsive">
|
||||
<table class="table table-hover align-middle mb-0" id="domainTable">
|
||||
<thead class="table-light">
|
||||
<tr>
|
||||
<th scope="col">Domain</th>
|
||||
<th scope="col">Bucket</th>
|
||||
<th scope="col" class="text-end">Actions</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for m in mappings %}
|
||||
<tr data-domain="{{ m.domain }}" data-bucket="{{ m.bucket }}">
|
||||
<td>
|
||||
<div class="d-flex align-items-center gap-2">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="text-success flex-shrink-0" viewBox="0 0 16 16">
|
||||
<path d="M0 8a8 8 0 1 1 16 0A8 8 0 0 1 0 8zm7.5-6.923c-.67.204-1.335.82-1.887 1.855A7.97 7.97 0 0 0 5.145 4H7.5V1.077zM4.09 4a9.267 9.267 0 0 1 .64-1.539 6.7 6.7 0 0 1 .597-.933A7.025 7.025 0 0 0 2.255 4H4.09zm-.582 3.5c.03-.877.138-1.718.312-2.5H1.674a6.958 6.958 0 0 0-.656 2.5h2.49zM4.847 5a12.5 12.5 0 0 0-.338 2.5H7.5V5H4.847zM8.5 5v2.5h2.99a12.495 12.495 0 0 0-.337-2.5H8.5zM4.51 8.5a12.5 12.5 0 0 0 .337 2.5H7.5V8.5H4.51zm3.99 0V11h2.653c.187-.765.306-1.608.338-2.5H8.5zM5.145 12c.138.386.295.744.468 1.068.552 1.035 1.218 1.65 1.887 1.855V12H5.145zm.182 2.472a6.696 6.696 0 0 1-.597-.933A9.268 9.268 0 0 1 4.09 12H2.255a7.024 7.024 0 0 0 3.072 2.472zM3.82 11a13.652 13.652 0 0 1-.312-2.5h-2.49c.062.89.291 1.733.656 2.5H3.82zm6.853 3.472A7.024 7.024 0 0 0 13.745 12H11.91a9.27 9.27 0 0 1-.64 1.539 6.688 6.688 0 0 1-.597.933zM8.5 12v2.923c.67-.204 1.335-.82 1.887-1.855.173-.324.33-.682.468-1.068H8.5zm3.68-1h2.146c.365-.767.594-1.61.656-2.5h-2.49a13.65 13.65 0 0 1-.312 2.5zm2.802-3.5a6.959 6.959 0 0 0-.656-2.5H12.18c.174.782.282 1.623.312 2.5h2.49zM11.27 2.461c.247.464.462.98.64 1.539h1.835a7.024 7.024 0 0 0-3.072-2.472c.218.284.418.598.597.933zM10.855 4a7.966 7.966 0 0 0-.468-1.068C9.835 1.897 9.17 1.282 8.5 1.077V4h2.355z"/>
|
||||
</svg>
|
||||
<div>
|
||||
<code class="fw-medium">{{ m.domain }}</code>
|
||||
<div class="text-muted small">http://{{ m.domain }}</div>
|
||||
</div>
|
||||
</div>
|
||||
</td>
|
||||
<td><span class="badge bg-primary bg-opacity-10 text-primary">{{ m.bucket }}</span></td>
|
||||
<td class="text-end">
|
||||
<div class="btn-group btn-group-sm" role="group">
|
||||
<button type="button" class="btn btn-outline-secondary"
|
||||
data-bs-toggle="modal"
|
||||
data-bs-target="#editDomainModal"
|
||||
data-domain="{{ m.domain }}"
|
||||
data-bucket="{{ m.bucket }}"
|
||||
title="Edit mapping">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" viewBox="0 0 16 16">
|
||||
<path d="M12.146.146a.5.5 0 0 1 .708 0l3 3a.5.5 0 0 1 0 .708l-10 10a.5.5 0 0 1-.168.11l-5 2a.5.5 0 0 1-.65-.65l2-5a.5.5 0 0 1 .11-.168l10-10zM11.207 2.5 13.5 4.793 14.793 3.5 12.5 1.207 11.207 2.5zm1.586 3L10.5 3.207 4 9.707V10h.5a.5.5 0 0 1 .5.5v.5h.5a.5.5 0 0 1 .5.5v.5h.293l6.5-6.5z"/>
|
||||
</svg>
|
||||
</button>
|
||||
<button type="button" class="btn btn-outline-danger"
|
||||
data-bs-toggle="modal"
|
||||
data-bs-target="#deleteDomainModal"
|
||||
data-domain="{{ m.domain }}"
|
||||
data-bucket="{{ m.bucket }}"
|
||||
title="Delete mapping">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" viewBox="0 0 16 16">
|
||||
<path d="M5.5 5.5A.5.5 0 0 1 6 6v6a.5.5 0 0 1-1 0V6a.5.5 0 0 1 .5-.5zm2.5 0a.5.5 0 0 1 .5.5v6a.5.5 0 0 1-1 0V6a.5.5 0 0 1 .5-.5zm3 .5a.5.5 0 0 0-1 0v6a.5.5 0 0 0 1 0V6z"/>
|
||||
<path fill-rule="evenodd" d="M14.5 3a1 1 0 0 1-1 1H13v9a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2V4h-.5a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1H6a1 1 0 0 1 1-1h2a1 1 0 0 1 1 1h3.5a1 1 0 0 1 1 1v1zM4.118 4 4 4.059V13a1 1 0 0 0 1 1h6a1 1 0 0 0 1-1V4.059L11.882 4H4.118zM2.5 3V2h11v1h-11z"/>
|
||||
</svg>
|
||||
</button>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<div id="noSearchResults" class="text-center py-4 d-none">
|
||||
<p class="text-muted mb-0">No mappings match your search.</p>
|
||||
</div>
|
||||
{% else %}
|
||||
<div class="empty-state text-center py-5">
|
||||
<div class="empty-state-icon mx-auto mb-3">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="48" height="48" fill="currentColor" class="text-muted" viewBox="0 0 16 16">
|
||||
<path d="M0 8a8 8 0 1 1 16 0A8 8 0 0 1 0 8zm7.5-6.923c-.67.204-1.335.82-1.887 1.855A7.97 7.97 0 0 0 5.145 4H7.5V1.077zM4.09 4a9.267 9.267 0 0 1 .64-1.539 6.7 6.7 0 0 1 .597-.933A7.025 7.025 0 0 0 2.255 4H4.09zm-.582 3.5c.03-.877.138-1.718.312-2.5H1.674a6.958 6.958 0 0 0-.656 2.5h2.49zM4.847 5a12.5 12.5 0 0 0-.338 2.5H7.5V5H4.847zM8.5 5v2.5h2.99a12.495 12.495 0 0 0-.337-2.5H8.5zM4.51 8.5a12.5 12.5 0 0 0 .337 2.5H7.5V8.5H4.51zm3.99 0V11h2.653c.187-.765.306-1.608.338-2.5H8.5zM5.145 12c.138.386.295.744.468 1.068.552 1.035 1.218 1.65 1.887 1.855V12H5.145zm.182 2.472a6.696 6.696 0 0 1-.597-.933A9.268 9.268 0 0 1 4.09 12H2.255a7.024 7.024 0 0 0 3.072 2.472zM3.82 11a13.652 13.652 0 0 1-.312-2.5h-2.49c.062.89.291 1.733.656 2.5H3.82zm6.853 3.472A7.024 7.024 0 0 0 13.745 12H11.91a9.27 9.27 0 0 1-.64 1.539 6.688 6.688 0 0 1-.597.933zM8.5 12v2.923c.67-.204 1.335-.82 1.887-1.855.173-.324.33-.682.468-1.068H8.5zm3.68-1h2.146c.365-.767.594-1.61.656-2.5h-2.49a13.65 13.65 0 0 1-.312 2.5zm2.802-3.5a6.959 6.959 0 0 0-.656-2.5H12.18c.174.782.282 1.623.312 2.5h2.49zM11.27 2.461c.247.464.462.98.64 1.539h1.835a7.024 7.024 0 0 0-3.072-2.472c.218.284.418.598.597.933zM10.855 4a7.966 7.966 0 0 0-.468-1.068C9.835 1.897 9.17 1.282 8.5 1.077V4h2.355z"/>
|
||||
</svg>
|
||||
</div>
|
||||
<h5 class="fw-semibold mb-2">No domain mappings yet</h5>
|
||||
<p class="text-muted mb-0">Add your first domain mapping to serve a bucket as a static website.</p>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="modal fade" id="editDomainModal" tabindex="-1" aria-hidden="true">
|
||||
<div class="modal-dialog modal-dialog-centered">
|
||||
<div class="modal-content">
|
||||
<div class="modal-header border-0 pb-0">
|
||||
<h5 class="modal-title fw-semibold">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" class="text-primary" viewBox="0 0 16 16">
|
||||
<path d="M12.146.146a.5.5 0 0 1 .708 0l3 3a.5.5 0 0 1 0 .708l-10 10a.5.5 0 0 1-.168.11l-5 2a.5.5 0 0 1-.65-.65l2-5a.5.5 0 0 1 .11-.168l10-10zM11.207 2.5 13.5 4.793 14.793 3.5 12.5 1.207 11.207 2.5zm1.586 3L10.5 3.207 4 9.707V10h.5a.5.5 0 0 1 .5.5v.5h.5a.5.5 0 0 1 .5.5v.5h.293l6.5-6.5zm-9.761 5.175-.106.106-1.528 3.821 3.821-1.528.106-.106A.5.5 0 0 1 5 12.5V12h-.5a.5.5 0 0 1-.5-.5V11h-.5a.5.5 0 0 1-.468-.325z"/>
|
||||
</svg>
|
||||
Edit Domain Mapping
|
||||
</h5>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
||||
</div>
|
||||
<form method="POST" id="editDomainForm">
|
||||
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
|
||||
<div class="modal-body">
|
||||
<div class="mb-3">
|
||||
<label class="form-label fw-medium">Domain</label>
|
||||
<input type="text" class="form-control bg-light" id="editDomainName" disabled>
|
||||
</div>
|
||||
<div class="mb-3">
|
||||
<label for="editBucket" class="form-label fw-medium">Bucket</label>
|
||||
{% if buckets %}
|
||||
<select class="form-select" id="editBucket" name="bucket" required>
|
||||
{% for b in buckets %}
|
||||
<option value="{{ b }}">{{ b }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
{% else %}
|
||||
<input type="text" class="form-control" id="editBucket" name="bucket" required>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="btn btn-outline-secondary" data-bs-dismiss="modal">Cancel</button>
|
||||
<button type="submit" class="btn btn-primary">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-1" viewBox="0 0 16 16">
|
||||
<path d="M10.97 4.97a.75.75 0 0 1 1.07 1.05l-3.99 4.99a.75.75 0 0 1-1.08.02L4.324 8.384a.75.75 0 1 1 1.06-1.06l2.094 2.093 3.473-4.425a.267.267 0 0 1 .02-.022z"/>
|
||||
</svg>
|
||||
Save
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="modal fade" id="deleteDomainModal" tabindex="-1" aria-hidden="true">
|
||||
<div class="modal-dialog modal-dialog-centered">
|
||||
<div class="modal-content">
|
||||
<form method="POST" id="deleteDomainForm">
|
||||
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
|
||||
<div class="modal-header border-0 pb-0">
|
||||
<h5 class="modal-title fw-semibold">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" fill="currentColor" class="text-danger" viewBox="0 0 16 16">
|
||||
<path d="M5.5 5.5A.5.5 0 0 1 6 6v6a.5.5 0 0 1-1 0V6a.5.5 0 0 1 .5-.5zm2.5 0a.5.5 0 0 1 .5.5v6a.5.5 0 0 1-1 0V6a.5.5 0 0 1 .5-.5zm3 .5a.5.5 0 0 0-1 0v6a.5.5 0 0 0 1 0V6z"/>
|
||||
<path fill-rule="evenodd" d="M14.5 3a1 1 0 0 1-1 1H13v9a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2V4h-.5a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1H6a1 1 0 0 1 1-1h2a1 1 0 0 1 1 1h3.5a1 1 0 0 1 1 1v1zM4.118 4 4 4.059V13a1 1 0 0 0 1 1h6a1 1 0 0 0 1-1V4.059L11.882 4H4.118zM2.5 3V2h11v1h-11z"/>
|
||||
</svg>
|
||||
Delete Domain Mapping
|
||||
</h5>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
||||
</div>
|
||||
<div class="modal-body">
|
||||
<p>Are you sure you want to delete the mapping for <strong><code id="deleteDomainName"></code></strong>?</p>
|
||||
<p class="text-muted small mb-0">Mapped to bucket: <code id="deleteBucketName"></code></p>
|
||||
<div class="alert alert-warning d-flex align-items-start small mt-3 mb-0" role="alert">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="flex-shrink-0 me-2 mt-0" viewBox="0 0 16 16">
|
||||
<path d="M8.982 1.566a1.13 1.13 0 0 0-1.96 0L.165 13.233c-.457.778.091 1.767.98 1.767h13.713c.889 0 1.438-.99.98-1.767L8.982 1.566zM8 5c.535 0 .954.462.9.995l-.35 3.507a.552.552 0 0 1-1.1 0L7.1 5.995A.905.905 0 0 1 8 5zm.002 6a1 1 0 1 1 0 2 1 1 0 0 1 0-2z"/>
|
||||
</svg>
|
||||
<div>This domain will stop serving website content immediately.</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="btn btn-outline-secondary" data-bs-dismiss="modal">Cancel</button>
|
||||
<button type="submit" class="btn btn-danger">
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" fill="currentColor" class="me-1" viewBox="0 0 16 16">
|
||||
<path d="M5.5 5.5A.5.5 0 0 1 6 6v6a.5.5 0 0 1-1 0V6a.5.5 0 0 1 .5-.5zm2.5 0a.5.5 0 0 1 .5.5v6a.5.5 0 0 1-1 0V6a.5.5 0 0 1 .5-.5zm3 .5a.5.5 0 0 0-1 0v6a.5.5 0 0 0 1 0V6z"/>
|
||||
<path fill-rule="evenodd" d="M14.5 3a1 1 0 0 1-1 1H13v9a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2V4h-.5a1 1 0 0 1-1-1V2a1 1 0 0 1 1-1H6a1 1 0 0 1 1-1h2a1 1 0 0 1 1 1h3.5a1 1 0 0 1 1 1v1zM4.118 4 4 4.059V13a1 1 0 0 0 1 1h6a1 1 0 0 0 1-1V4.059L11.882 4H4.118zM2.5 3V2h11v1h-11z"/>
|
||||
</svg>
|
||||
Delete
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% endblock %}
|
||||
|
||||
{% block extra_scripts %}
|
||||
<script>
|
||||
(function () {
|
||||
function normalizeDomain(val) {
|
||||
val = val.trim().toLowerCase();
|
||||
if (val.indexOf('https://') === 0) val = val.substring(8);
|
||||
else if (val.indexOf('http://') === 0) val = val.substring(7);
|
||||
var slashIdx = val.indexOf('/');
|
||||
if (slashIdx !== -1) val = val.substring(0, slashIdx);
|
||||
var qIdx = val.indexOf('?');
|
||||
if (qIdx !== -1) val = val.substring(0, qIdx);
|
||||
var hIdx = val.indexOf('#');
|
||||
if (hIdx !== -1) val = val.substring(0, hIdx);
|
||||
var colonIdx = val.indexOf(':');
|
||||
if (colonIdx !== -1) val = val.substring(0, colonIdx);
|
||||
return val;
|
||||
}
|
||||
|
||||
var domainInput = document.getElementById('domain');
|
||||
var preview = document.getElementById('domainPreview');
|
||||
var previewUrl = document.getElementById('domainPreviewUrl');
|
||||
if (domainInput && preview) {
|
||||
domainInput.addEventListener('input', function () {
|
||||
var clean = normalizeDomain(this.value);
|
||||
if (clean && /^[a-z0-9]([a-z0-9\-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9\-]*[a-z0-9])?)*$/.test(clean)) {
|
||||
previewUrl.textContent = 'http://' + clean;
|
||||
preview.classList.remove('d-none');
|
||||
} else {
|
||||
preview.classList.add('d-none');
|
||||
}
|
||||
});
|
||||
|
||||
var createForm = document.getElementById('createDomainForm');
|
||||
if (createForm) {
|
||||
createForm.addEventListener('submit', function () {
|
||||
domainInput.value = normalizeDomain(domainInput.value);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
var editModal = document.getElementById('editDomainModal');
|
||||
if (editModal) {
|
||||
editModal.addEventListener('show.bs.modal', function (event) {
|
||||
var btn = event.relatedTarget;
|
||||
var domain = btn.getAttribute('data-domain');
|
||||
var bucket = btn.getAttribute('data-bucket');
|
||||
document.getElementById('editDomainName').value = domain;
|
||||
var editBucket = document.getElementById('editBucket');
|
||||
editBucket.value = bucket;
|
||||
document.getElementById('editDomainForm').action = '{{ url_for("ui.update_website_domain", domain="__DOMAIN__") }}'.replace('__DOMAIN__', encodeURIComponent(domain));
|
||||
});
|
||||
}
|
||||
|
||||
var deleteModal = document.getElementById('deleteDomainModal');
|
||||
if (deleteModal) {
|
||||
deleteModal.addEventListener('show.bs.modal', function (event) {
|
||||
var btn = event.relatedTarget;
|
||||
var domain = btn.getAttribute('data-domain');
|
||||
var bucket = btn.getAttribute('data-bucket') || '';
|
||||
document.getElementById('deleteDomainName').textContent = domain;
|
||||
document.getElementById('deleteBucketName').textContent = bucket;
|
||||
document.getElementById('deleteDomainForm').action = '{{ url_for("ui.delete_website_domain", domain="__DOMAIN__") }}'.replace('__DOMAIN__', encodeURIComponent(domain));
|
||||
});
|
||||
}
|
||||
|
||||
var searchInput = document.getElementById('domainSearch');
|
||||
if (searchInput) {
|
||||
searchInput.addEventListener('input', function () {
|
||||
var q = this.value.toLowerCase();
|
||||
var rows = document.querySelectorAll('#domainTable tbody tr');
|
||||
var visible = 0;
|
||||
rows.forEach(function (row) {
|
||||
var domain = (row.getAttribute('data-domain') || '').toLowerCase();
|
||||
var bucket = (row.getAttribute('data-bucket') || '').toLowerCase();
|
||||
var match = !q || domain.indexOf(q) !== -1 || bucket.indexOf(q) !== -1;
|
||||
row.style.display = match ? '' : 'none';
|
||||
if (match) visible++;
|
||||
});
|
||||
var noResults = document.getElementById('noSearchResults');
|
||||
if (noResults) {
|
||||
noResults.classList.toggle('d-none', visible > 0);
|
||||
}
|
||||
});
|
||||
}
|
||||
})();
|
||||
</script>
|
||||
{% endblock %}
|
||||
@@ -35,6 +35,7 @@ def app(tmp_path: Path):
|
||||
flask_app = create_api_app(
|
||||
{
|
||||
"TESTING": True,
|
||||
"SECRET_KEY": "testing",
|
||||
"STORAGE_ROOT": storage_root,
|
||||
"IAM_CONFIG": iam_config,
|
||||
"BUCKET_POLICY_PATH": bucket_policies,
|
||||
|
||||
@@ -1,6 +1,3 @@
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
|
||||
def test_bucket_and_object_lifecycle(client, signer):
|
||||
headers = signer("PUT", "/photos")
|
||||
response = client.put("/photos", headers=headers)
|
||||
@@ -104,12 +101,12 @@ def test_request_id_header_present(client, signer):
|
||||
assert response.headers.get("X-Request-ID")
|
||||
|
||||
|
||||
def test_healthcheck_returns_version(client):
|
||||
response = client.get("/healthz")
|
||||
def test_healthcheck_returns_status(client):
|
||||
response = client.get("/myfsio/health")
|
||||
data = response.get_json()
|
||||
assert response.status_code == 200
|
||||
assert data["status"] == "ok"
|
||||
assert "version" in data
|
||||
assert "version" not in data
|
||||
|
||||
|
||||
def test_missing_credentials_denied(client):
|
||||
@@ -117,36 +114,20 @@ def test_missing_credentials_denied(client):
|
||||
assert response.status_code == 403
|
||||
|
||||
|
||||
def test_presign_and_bucket_policies(client, signer):
|
||||
# Create bucket and object
|
||||
def test_bucket_policies_deny_reads(client, signer):
|
||||
import json
|
||||
|
||||
headers = signer("PUT", "/docs")
|
||||
assert client.put("/docs", headers=headers).status_code == 200
|
||||
|
||||
|
||||
headers = signer("PUT", "/docs/readme.txt", body=b"content")
|
||||
assert client.put("/docs/readme.txt", headers=headers, data=b"content").status_code == 200
|
||||
|
||||
# Generate presigned GET URL and follow it
|
||||
json_body = {"method": "GET", "expires_in": 120}
|
||||
# Flask test client json parameter automatically sets Content-Type and serializes body
|
||||
# But for signing we need the body bytes.
|
||||
import json
|
||||
body_bytes = json.dumps(json_body).encode("utf-8")
|
||||
headers = signer("POST", "/presign/docs/readme.txt", headers={"Content-Type": "application/json"}, body=body_bytes)
|
||||
|
||||
response = client.post(
|
||||
"/presign/docs/readme.txt",
|
||||
headers=headers,
|
||||
json=json_body,
|
||||
)
|
||||
headers = signer("GET", "/docs/readme.txt")
|
||||
response = client.get("/docs/readme.txt", headers=headers)
|
||||
assert response.status_code == 200
|
||||
presigned_url = response.get_json()["url"]
|
||||
parts = urlsplit(presigned_url)
|
||||
presigned_path = f"{parts.path}?{parts.query}"
|
||||
download = client.get(presigned_path)
|
||||
assert download.status_code == 200
|
||||
assert download.data == b"content"
|
||||
assert response.data == b"content"
|
||||
|
||||
# Attach a deny policy for GETs
|
||||
policy = {
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
@@ -160,29 +141,26 @@ def test_presign_and_bucket_policies(client, signer):
|
||||
],
|
||||
}
|
||||
policy_bytes = json.dumps(policy).encode("utf-8")
|
||||
headers = signer("PUT", "/bucket-policy/docs", headers={"Content-Type": "application/json"}, body=policy_bytes)
|
||||
assert client.put("/bucket-policy/docs", headers=headers, json=policy).status_code == 204
|
||||
|
||||
headers = signer("GET", "/bucket-policy/docs")
|
||||
fetched = client.get("/bucket-policy/docs", headers=headers)
|
||||
headers = signer("PUT", "/docs?policy", headers={"Content-Type": "application/json"}, body=policy_bytes)
|
||||
assert client.put("/docs?policy", headers=headers, json=policy).status_code == 204
|
||||
|
||||
headers = signer("GET", "/docs?policy")
|
||||
fetched = client.get("/docs?policy", headers=headers)
|
||||
assert fetched.status_code == 200
|
||||
assert fetched.get_json()["Version"] == "2012-10-17"
|
||||
|
||||
# Reads are now denied by bucket policy
|
||||
headers = signer("GET", "/docs/readme.txt")
|
||||
denied = client.get("/docs/readme.txt", headers=headers)
|
||||
assert denied.status_code == 403
|
||||
|
||||
# Presign attempts are also denied
|
||||
json_body = {"method": "GET", "expires_in": 60}
|
||||
body_bytes = json.dumps(json_body).encode("utf-8")
|
||||
headers = signer("POST", "/presign/docs/readme.txt", headers={"Content-Type": "application/json"}, body=body_bytes)
|
||||
response = client.post(
|
||||
"/presign/docs/readme.txt",
|
||||
headers=headers,
|
||||
json=json_body,
|
||||
)
|
||||
assert response.status_code == 403
|
||||
headers = signer("DELETE", "/docs?policy")
|
||||
assert client.delete("/docs?policy", headers=headers).status_code == 204
|
||||
|
||||
headers = signer("DELETE", "/docs/readme.txt")
|
||||
assert client.delete("/docs/readme.txt", headers=headers).status_code == 204
|
||||
|
||||
headers = signer("DELETE", "/docs")
|
||||
assert client.delete("/docs", headers=headers).status_code == 204
|
||||
|
||||
|
||||
def test_trailing_slash_returns_xml(client):
|
||||
@@ -193,9 +171,11 @@ def test_trailing_slash_returns_xml(client):
|
||||
|
||||
|
||||
def test_public_policy_allows_anonymous_list_and_read(client, signer):
|
||||
import json
|
||||
|
||||
headers = signer("PUT", "/public")
|
||||
assert client.put("/public", headers=headers).status_code == 200
|
||||
|
||||
|
||||
headers = signer("PUT", "/public/hello.txt", body=b"hi")
|
||||
assert client.put("/public/hello.txt", headers=headers, data=b"hi").status_code == 200
|
||||
|
||||
@@ -221,10 +201,9 @@ def test_public_policy_allows_anonymous_list_and_read(client, signer):
|
||||
},
|
||||
],
|
||||
}
|
||||
import json
|
||||
policy_bytes = json.dumps(policy).encode("utf-8")
|
||||
headers = signer("PUT", "/bucket-policy/public", headers={"Content-Type": "application/json"}, body=policy_bytes)
|
||||
assert client.put("/bucket-policy/public", headers=headers, json=policy).status_code == 204
|
||||
headers = signer("PUT", "/public?policy", headers={"Content-Type": "application/json"}, body=policy_bytes)
|
||||
assert client.put("/public?policy", headers=headers, json=policy).status_code == 204
|
||||
|
||||
list_response = client.get("/public")
|
||||
assert list_response.status_code == 200
|
||||
@@ -236,18 +215,20 @@ def test_public_policy_allows_anonymous_list_and_read(client, signer):
|
||||
|
||||
headers = signer("DELETE", "/public/hello.txt")
|
||||
assert client.delete("/public/hello.txt", headers=headers).status_code == 204
|
||||
|
||||
headers = signer("DELETE", "/bucket-policy/public")
|
||||
assert client.delete("/bucket-policy/public", headers=headers).status_code == 204
|
||||
|
||||
|
||||
headers = signer("DELETE", "/public?policy")
|
||||
assert client.delete("/public?policy", headers=headers).status_code == 204
|
||||
|
||||
headers = signer("DELETE", "/public")
|
||||
assert client.delete("/public", headers=headers).status_code == 204
|
||||
|
||||
|
||||
def test_principal_dict_with_object_get_only(client, signer):
|
||||
import json
|
||||
|
||||
headers = signer("PUT", "/mixed")
|
||||
assert client.put("/mixed", headers=headers).status_code == 200
|
||||
|
||||
|
||||
headers = signer("PUT", "/mixed/only.txt", body=b"ok")
|
||||
assert client.put("/mixed/only.txt", headers=headers, data=b"ok").status_code == 200
|
||||
|
||||
@@ -270,10 +251,9 @@ def test_principal_dict_with_object_get_only(client, signer):
|
||||
},
|
||||
],
|
||||
}
|
||||
import json
|
||||
policy_bytes = json.dumps(policy).encode("utf-8")
|
||||
headers = signer("PUT", "/bucket-policy/mixed", headers={"Content-Type": "application/json"}, body=policy_bytes)
|
||||
assert client.put("/bucket-policy/mixed", headers=headers, json=policy).status_code == 204
|
||||
headers = signer("PUT", "/mixed?policy", headers={"Content-Type": "application/json"}, body=policy_bytes)
|
||||
assert client.put("/mixed?policy", headers=headers, json=policy).status_code == 204
|
||||
|
||||
assert client.get("/mixed").status_code == 403
|
||||
allowed = client.get("/mixed/only.txt")
|
||||
@@ -282,18 +262,20 @@ def test_principal_dict_with_object_get_only(client, signer):
|
||||
|
||||
headers = signer("DELETE", "/mixed/only.txt")
|
||||
assert client.delete("/mixed/only.txt", headers=headers).status_code == 204
|
||||
|
||||
headers = signer("DELETE", "/bucket-policy/mixed")
|
||||
assert client.delete("/bucket-policy/mixed", headers=headers).status_code == 204
|
||||
|
||||
|
||||
headers = signer("DELETE", "/mixed?policy")
|
||||
assert client.delete("/mixed?policy", headers=headers).status_code == 204
|
||||
|
||||
headers = signer("DELETE", "/mixed")
|
||||
assert client.delete("/mixed", headers=headers).status_code == 204
|
||||
|
||||
|
||||
def test_bucket_policy_wildcard_resource_allows_object_get(client, signer):
|
||||
import json
|
||||
|
||||
headers = signer("PUT", "/test")
|
||||
assert client.put("/test", headers=headers).status_code == 200
|
||||
|
||||
|
||||
headers = signer("PUT", "/test/vid.mp4", body=b"video")
|
||||
assert client.put("/test/vid.mp4", headers=headers, data=b"video").status_code == 200
|
||||
|
||||
@@ -314,10 +296,9 @@ def test_bucket_policy_wildcard_resource_allows_object_get(client, signer):
|
||||
},
|
||||
],
|
||||
}
|
||||
import json
|
||||
policy_bytes = json.dumps(policy).encode("utf-8")
|
||||
headers = signer("PUT", "/bucket-policy/test", headers={"Content-Type": "application/json"}, body=policy_bytes)
|
||||
assert client.put("/bucket-policy/test", headers=headers, json=policy).status_code == 204
|
||||
headers = signer("PUT", "/test?policy", headers={"Content-Type": "application/json"}, body=policy_bytes)
|
||||
assert client.put("/test?policy", headers=headers, json=policy).status_code == 204
|
||||
|
||||
listing = client.get("/test")
|
||||
assert listing.status_code == 403
|
||||
@@ -327,10 +308,10 @@ def test_bucket_policy_wildcard_resource_allows_object_get(client, signer):
|
||||
|
||||
headers = signer("DELETE", "/test/vid.mp4")
|
||||
assert client.delete("/test/vid.mp4", headers=headers).status_code == 204
|
||||
|
||||
headers = signer("DELETE", "/bucket-policy/test")
|
||||
assert client.delete("/bucket-policy/test", headers=headers).status_code == 204
|
||||
|
||||
|
||||
headers = signer("DELETE", "/test?policy")
|
||||
assert client.delete("/test?policy", headers=headers).status_code == 204
|
||||
|
||||
headers = signer("DELETE", "/test")
|
||||
assert client.delete("/test", headers=headers).status_code == 204
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ def kms_client(tmp_path):
|
||||
|
||||
app = create_app({
|
||||
"TESTING": True,
|
||||
"SECRET_KEY": "testing",
|
||||
"STORAGE_ROOT": str(tmp_path / "storage"),
|
||||
"IAM_CONFIG": str(tmp_path / "iam.json"),
|
||||
"BUCKET_POLICY_PATH": str(tmp_path / "policies.json"),
|
||||
|
||||
@@ -321,8 +321,9 @@ class TestNotificationService:
|
||||
assert "events_sent" in stats
|
||||
assert "events_failed" in stats
|
||||
|
||||
@patch("app.notifications.requests.post")
|
||||
def test_send_notification_success(self, mock_post, notification_service):
|
||||
@patch("app.notifications._pinned_post")
|
||||
@patch("app.notifications._resolve_and_check_url", return_value="93.184.216.34")
|
||||
def test_send_notification_success(self, mock_resolve, mock_post, notification_service):
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_post.return_value = mock_response
|
||||
@@ -337,8 +338,9 @@ class TestNotificationService:
|
||||
notification_service._send_notification(event, destination)
|
||||
mock_post.assert_called_once()
|
||||
|
||||
@patch("app.notifications.requests.post")
|
||||
def test_send_notification_retry_on_failure(self, mock_post, notification_service):
|
||||
@patch("app.notifications._pinned_post")
|
||||
@patch("app.notifications._resolve_and_check_url", return_value="93.184.216.34")
|
||||
def test_send_notification_retry_on_failure(self, mock_resolve, mock_post, notification_service):
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 500
|
||||
mock_response.text = "Internal Server Error"
|
||||
|
||||
297
tests/test_operation_metrics.py
Normal file
297
tests/test_operation_metrics.py
Normal file
@@ -0,0 +1,297 @@
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.operation_metrics import (
|
||||
OperationMetricsCollector,
|
||||
OperationStats,
|
||||
classify_endpoint,
|
||||
)
|
||||
|
||||
|
||||
class TestOperationStats:
|
||||
def test_initial_state(self):
|
||||
stats = OperationStats()
|
||||
assert stats.count == 0
|
||||
assert stats.success_count == 0
|
||||
assert stats.error_count == 0
|
||||
assert stats.latency_sum_ms == 0.0
|
||||
assert stats.bytes_in == 0
|
||||
assert stats.bytes_out == 0
|
||||
|
||||
def test_record_success(self):
|
||||
stats = OperationStats()
|
||||
stats.record(latency_ms=50.0, success=True, bytes_in=100, bytes_out=200)
|
||||
|
||||
assert stats.count == 1
|
||||
assert stats.success_count == 1
|
||||
assert stats.error_count == 0
|
||||
assert stats.latency_sum_ms == 50.0
|
||||
assert stats.latency_min_ms == 50.0
|
||||
assert stats.latency_max_ms == 50.0
|
||||
assert stats.bytes_in == 100
|
||||
assert stats.bytes_out == 200
|
||||
|
||||
def test_record_error(self):
|
||||
stats = OperationStats()
|
||||
stats.record(latency_ms=100.0, success=False, bytes_in=50, bytes_out=0)
|
||||
|
||||
assert stats.count == 1
|
||||
assert stats.success_count == 0
|
||||
assert stats.error_count == 1
|
||||
|
||||
def test_latency_min_max(self):
|
||||
stats = OperationStats()
|
||||
stats.record(latency_ms=50.0, success=True)
|
||||
stats.record(latency_ms=10.0, success=True)
|
||||
stats.record(latency_ms=100.0, success=True)
|
||||
|
||||
assert stats.latency_min_ms == 10.0
|
||||
assert stats.latency_max_ms == 100.0
|
||||
assert stats.latency_sum_ms == 160.0
|
||||
|
||||
def test_to_dict(self):
|
||||
stats = OperationStats()
|
||||
stats.record(latency_ms=50.0, success=True, bytes_in=100, bytes_out=200)
|
||||
stats.record(latency_ms=100.0, success=False, bytes_in=50, bytes_out=0)
|
||||
|
||||
result = stats.to_dict()
|
||||
assert result["count"] == 2
|
||||
assert result["success_count"] == 1
|
||||
assert result["error_count"] == 1
|
||||
assert result["latency_avg_ms"] == 75.0
|
||||
assert result["latency_min_ms"] == 50.0
|
||||
assert result["latency_max_ms"] == 100.0
|
||||
assert result["bytes_in"] == 150
|
||||
assert result["bytes_out"] == 200
|
||||
|
||||
def test_to_dict_empty(self):
|
||||
stats = OperationStats()
|
||||
result = stats.to_dict()
|
||||
assert result["count"] == 0
|
||||
assert result["latency_avg_ms"] == 0.0
|
||||
assert result["latency_min_ms"] == 0.0
|
||||
|
||||
def test_merge(self):
|
||||
stats1 = OperationStats()
|
||||
stats1.record(latency_ms=50.0, success=True, bytes_in=100, bytes_out=200)
|
||||
|
||||
stats2 = OperationStats()
|
||||
stats2.record(latency_ms=10.0, success=True, bytes_in=50, bytes_out=100)
|
||||
stats2.record(latency_ms=100.0, success=False, bytes_in=25, bytes_out=50)
|
||||
|
||||
stats1.merge(stats2)
|
||||
|
||||
assert stats1.count == 3
|
||||
assert stats1.success_count == 2
|
||||
assert stats1.error_count == 1
|
||||
assert stats1.latency_min_ms == 10.0
|
||||
assert stats1.latency_max_ms == 100.0
|
||||
assert stats1.bytes_in == 175
|
||||
assert stats1.bytes_out == 350
|
||||
|
||||
|
||||
class TestClassifyEndpoint:
|
||||
def test_root_path(self):
|
||||
assert classify_endpoint("/") == "service"
|
||||
assert classify_endpoint("") == "service"
|
||||
|
||||
def test_ui_paths(self):
|
||||
assert classify_endpoint("/ui") == "ui"
|
||||
assert classify_endpoint("/ui/buckets") == "ui"
|
||||
assert classify_endpoint("/ui/metrics") == "ui"
|
||||
|
||||
def test_kms_paths(self):
|
||||
assert classify_endpoint("/kms") == "kms"
|
||||
assert classify_endpoint("/kms/keys") == "kms"
|
||||
|
||||
def test_service_paths(self):
|
||||
assert classify_endpoint("/myfsio/health") == "service"
|
||||
|
||||
def test_bucket_paths(self):
|
||||
assert classify_endpoint("/mybucket") == "bucket"
|
||||
assert classify_endpoint("/mybucket/") == "bucket"
|
||||
|
||||
def test_object_paths(self):
|
||||
assert classify_endpoint("/mybucket/mykey") == "object"
|
||||
assert classify_endpoint("/mybucket/folder/nested/key.txt") == "object"
|
||||
|
||||
|
||||
class TestOperationMetricsCollector:
|
||||
def test_record_and_get_stats(self, tmp_path: Path):
|
||||
collector = OperationMetricsCollector(
|
||||
storage_root=tmp_path,
|
||||
interval_minutes=60,
|
||||
retention_hours=24,
|
||||
)
|
||||
|
||||
try:
|
||||
collector.record_request(
|
||||
method="GET",
|
||||
endpoint_type="bucket",
|
||||
status_code=200,
|
||||
latency_ms=50.0,
|
||||
bytes_in=0,
|
||||
bytes_out=1000,
|
||||
)
|
||||
|
||||
collector.record_request(
|
||||
method="PUT",
|
||||
endpoint_type="object",
|
||||
status_code=201,
|
||||
latency_ms=100.0,
|
||||
bytes_in=500,
|
||||
bytes_out=0,
|
||||
)
|
||||
|
||||
collector.record_request(
|
||||
method="GET",
|
||||
endpoint_type="object",
|
||||
status_code=404,
|
||||
latency_ms=25.0,
|
||||
bytes_in=0,
|
||||
bytes_out=0,
|
||||
error_code="NoSuchKey",
|
||||
)
|
||||
|
||||
stats = collector.get_current_stats()
|
||||
|
||||
assert stats["totals"]["count"] == 3
|
||||
assert stats["totals"]["success_count"] == 2
|
||||
assert stats["totals"]["error_count"] == 1
|
||||
|
||||
assert "GET" in stats["by_method"]
|
||||
assert stats["by_method"]["GET"]["count"] == 2
|
||||
assert "PUT" in stats["by_method"]
|
||||
assert stats["by_method"]["PUT"]["count"] == 1
|
||||
|
||||
assert "bucket" in stats["by_endpoint"]
|
||||
assert "object" in stats["by_endpoint"]
|
||||
assert stats["by_endpoint"]["object"]["count"] == 2
|
||||
|
||||
assert stats["by_status_class"]["2xx"] == 2
|
||||
assert stats["by_status_class"]["4xx"] == 1
|
||||
|
||||
assert stats["error_codes"]["NoSuchKey"] == 1
|
||||
finally:
|
||||
collector.shutdown()
|
||||
|
||||
def test_thread_safety(self, tmp_path: Path):
|
||||
collector = OperationMetricsCollector(
|
||||
storage_root=tmp_path,
|
||||
interval_minutes=60,
|
||||
retention_hours=24,
|
||||
)
|
||||
|
||||
try:
|
||||
num_threads = 5
|
||||
requests_per_thread = 100
|
||||
threads = []
|
||||
|
||||
def record_requests():
|
||||
for _ in range(requests_per_thread):
|
||||
collector.record_request(
|
||||
method="GET",
|
||||
endpoint_type="object",
|
||||
status_code=200,
|
||||
latency_ms=10.0,
|
||||
)
|
||||
|
||||
for _ in range(num_threads):
|
||||
t = threading.Thread(target=record_requests)
|
||||
threads.append(t)
|
||||
t.start()
|
||||
|
||||
for t in threads:
|
||||
t.join()
|
||||
|
||||
stats = collector.get_current_stats()
|
||||
assert stats["totals"]["count"] == num_threads * requests_per_thread
|
||||
finally:
|
||||
collector.shutdown()
|
||||
|
||||
def test_status_class_categorization(self, tmp_path: Path):
|
||||
collector = OperationMetricsCollector(
|
||||
storage_root=tmp_path,
|
||||
interval_minutes=60,
|
||||
retention_hours=24,
|
||||
)
|
||||
|
||||
try:
|
||||
collector.record_request("GET", "object", 200, 10.0)
|
||||
collector.record_request("GET", "object", 204, 10.0)
|
||||
collector.record_request("GET", "object", 301, 10.0)
|
||||
collector.record_request("GET", "object", 304, 10.0)
|
||||
collector.record_request("GET", "object", 400, 10.0)
|
||||
collector.record_request("GET", "object", 403, 10.0)
|
||||
collector.record_request("GET", "object", 404, 10.0)
|
||||
collector.record_request("GET", "object", 500, 10.0)
|
||||
collector.record_request("GET", "object", 503, 10.0)
|
||||
|
||||
stats = collector.get_current_stats()
|
||||
assert stats["by_status_class"]["2xx"] == 2
|
||||
assert stats["by_status_class"]["3xx"] == 2
|
||||
assert stats["by_status_class"]["4xx"] == 3
|
||||
assert stats["by_status_class"]["5xx"] == 2
|
||||
finally:
|
||||
collector.shutdown()
|
||||
|
||||
def test_error_code_tracking(self, tmp_path: Path):
|
||||
collector = OperationMetricsCollector(
|
||||
storage_root=tmp_path,
|
||||
interval_minutes=60,
|
||||
retention_hours=24,
|
||||
)
|
||||
|
||||
try:
|
||||
collector.record_request("GET", "object", 404, 10.0, error_code="NoSuchKey")
|
||||
collector.record_request("GET", "object", 404, 10.0, error_code="NoSuchKey")
|
||||
collector.record_request("GET", "bucket", 403, 10.0, error_code="AccessDenied")
|
||||
collector.record_request("PUT", "object", 500, 10.0, error_code="InternalError")
|
||||
|
||||
stats = collector.get_current_stats()
|
||||
assert stats["error_codes"]["NoSuchKey"] == 2
|
||||
assert stats["error_codes"]["AccessDenied"] == 1
|
||||
assert stats["error_codes"]["InternalError"] == 1
|
||||
finally:
|
||||
collector.shutdown()
|
||||
|
||||
def test_history_persistence(self, tmp_path: Path):
|
||||
collector = OperationMetricsCollector(
|
||||
storage_root=tmp_path,
|
||||
interval_minutes=60,
|
||||
retention_hours=24,
|
||||
)
|
||||
|
||||
try:
|
||||
collector.record_request("GET", "object", 200, 10.0)
|
||||
collector._take_snapshot()
|
||||
|
||||
history = collector.get_history()
|
||||
assert len(history) == 1
|
||||
assert history[0]["totals"]["count"] == 1
|
||||
|
||||
config_path = tmp_path / ".myfsio.sys" / "config" / "operation_metrics.json"
|
||||
assert config_path.exists()
|
||||
finally:
|
||||
collector.shutdown()
|
||||
|
||||
def test_get_history_with_hours_filter(self, tmp_path: Path):
|
||||
collector = OperationMetricsCollector(
|
||||
storage_root=tmp_path,
|
||||
interval_minutes=60,
|
||||
retention_hours=24,
|
||||
)
|
||||
|
||||
try:
|
||||
collector.record_request("GET", "object", 200, 10.0)
|
||||
collector._take_snapshot()
|
||||
|
||||
history_all = collector.get_history()
|
||||
history_recent = collector.get_history(hours=1)
|
||||
|
||||
assert len(history_all) >= len(history_recent)
|
||||
finally:
|
||||
collector.shutdown()
|
||||
@@ -1,191 +0,0 @@
|
||||
import hashlib
|
||||
import hmac
|
||||
import pytest
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from urllib.parse import quote
|
||||
|
||||
def _sign(key, msg):
|
||||
return hmac.new(key, msg.encode("utf-8"), hashlib.sha256).digest()
|
||||
|
||||
def _get_signature_key(key, date_stamp, region_name, service_name):
|
||||
k_date = _sign(("AWS4" + key).encode("utf-8"), date_stamp)
|
||||
k_region = _sign(k_date, region_name)
|
||||
k_service = _sign(k_region, service_name)
|
||||
k_signing = _sign(k_service, "aws4_request")
|
||||
return k_signing
|
||||
|
||||
def create_signed_headers(
|
||||
method,
|
||||
path,
|
||||
headers=None,
|
||||
body=None,
|
||||
access_key="test",
|
||||
secret_key="secret",
|
||||
region="us-east-1",
|
||||
service="s3",
|
||||
timestamp=None
|
||||
):
|
||||
if headers is None:
|
||||
headers = {}
|
||||
|
||||
if timestamp is None:
|
||||
now = datetime.now(timezone.utc)
|
||||
else:
|
||||
now = timestamp
|
||||
|
||||
amz_date = now.strftime("%Y%m%dT%H%M%SZ")
|
||||
date_stamp = now.strftime("%Y%m%d")
|
||||
|
||||
headers["X-Amz-Date"] = amz_date
|
||||
headers["Host"] = "testserver"
|
||||
|
||||
canonical_uri = quote(path, safe="/-_.~")
|
||||
canonical_query_string = ""
|
||||
|
||||
canonical_headers = ""
|
||||
signed_headers_list = []
|
||||
for k, v in sorted(headers.items(), key=lambda x: x[0].lower()):
|
||||
canonical_headers += f"{k.lower()}:{v.strip()}\n"
|
||||
signed_headers_list.append(k.lower())
|
||||
|
||||
signed_headers = ";".join(signed_headers_list)
|
||||
|
||||
payload_hash = hashlib.sha256(body or b"").hexdigest()
|
||||
headers["X-Amz-Content-Sha256"] = payload_hash
|
||||
|
||||
canonical_request = f"{method}\n{canonical_uri}\n{canonical_query_string}\n{canonical_headers}\n{signed_headers}\n{payload_hash}"
|
||||
|
||||
credential_scope = f"{date_stamp}/{region}/{service}/aws4_request"
|
||||
string_to_sign = f"AWS4-HMAC-SHA256\n{amz_date}\n{credential_scope}\n{hashlib.sha256(canonical_request.encode('utf-8')).hexdigest()}"
|
||||
|
||||
signing_key = _get_signature_key(secret_key, date_stamp, region, service)
|
||||
signature = hmac.new(signing_key, string_to_sign.encode("utf-8"), hashlib.sha256).hexdigest()
|
||||
|
||||
headers["Authorization"] = (
|
||||
f"AWS4-HMAC-SHA256 Credential={access_key}/{credential_scope}, "
|
||||
f"SignedHeaders={signed_headers}, Signature={signature}"
|
||||
)
|
||||
return headers
|
||||
|
||||
def test_sigv4_old_date(client):
|
||||
# Test with a date 20 minutes in the past
|
||||
old_time = datetime.now(timezone.utc) - timedelta(minutes=20)
|
||||
headers = create_signed_headers("GET", "/", timestamp=old_time)
|
||||
|
||||
response = client.get("/", headers=headers)
|
||||
assert response.status_code == 403
|
||||
assert b"Request timestamp too old" in response.data
|
||||
|
||||
def test_sigv4_future_date(client):
|
||||
# Test with a date 20 minutes in the future
|
||||
future_time = datetime.now(timezone.utc) + timedelta(minutes=20)
|
||||
headers = create_signed_headers("GET", "/", timestamp=future_time)
|
||||
|
||||
response = client.get("/", headers=headers)
|
||||
assert response.status_code == 403
|
||||
assert b"Request timestamp too old" in response.data # The error message is the same
|
||||
|
||||
def test_path_traversal_in_key(client, signer):
|
||||
headers = signer("PUT", "/test-bucket")
|
||||
client.put("/test-bucket", headers=headers)
|
||||
|
||||
# Try to upload with .. in key
|
||||
headers = signer("PUT", "/test-bucket/../secret.txt", body=b"attack")
|
||||
response = client.put("/test-bucket/../secret.txt", headers=headers, data=b"attack")
|
||||
|
||||
# Should be rejected by storage layer or flask routing
|
||||
# Flask might normalize it before it reaches the app, but if it reaches, it should fail.
|
||||
# If Flask normalizes /test-bucket/../secret.txt to /secret.txt, then it hits 404 (bucket not found) or 403.
|
||||
# But we want to test the storage layer check.
|
||||
# We can try to encode the dots?
|
||||
|
||||
# If we use a key that doesn't get normalized by Flask routing easily.
|
||||
# But wait, the route is /<bucket_name>/<path:object_key>
|
||||
# If I send /test-bucket/folder/../file.txt, Flask might pass "folder/../file.txt" as object_key?
|
||||
# Let's try.
|
||||
|
||||
headers = signer("PUT", "/test-bucket/folder/../file.txt", body=b"attack")
|
||||
response = client.put("/test-bucket/folder/../file.txt", headers=headers, data=b"attack")
|
||||
|
||||
# If Flask normalizes it, it becomes /test-bucket/file.txt.
|
||||
# If it doesn't, it hits our check.
|
||||
|
||||
# Let's try to call the storage method directly to verify the check works,
|
||||
# because testing via client depends on Flask's URL handling.
|
||||
pass
|
||||
|
||||
def test_storage_path_traversal(app):
|
||||
storage = app.extensions["object_storage"]
|
||||
from app.storage import StorageError, ObjectStorage
|
||||
from app.encrypted_storage import EncryptedObjectStorage
|
||||
|
||||
# Get the underlying ObjectStorage if wrapped
|
||||
if isinstance(storage, EncryptedObjectStorage):
|
||||
storage = storage.storage
|
||||
|
||||
with pytest.raises(StorageError, match="Object key contains parent directory references"):
|
||||
storage._sanitize_object_key("folder/../file.txt")
|
||||
|
||||
with pytest.raises(StorageError, match="Object key contains parent directory references"):
|
||||
storage._sanitize_object_key("..")
|
||||
|
||||
def test_head_bucket(client, signer):
|
||||
headers = signer("PUT", "/head-test")
|
||||
client.put("/head-test", headers=headers)
|
||||
|
||||
headers = signer("HEAD", "/head-test")
|
||||
response = client.head("/head-test", headers=headers)
|
||||
assert response.status_code == 200
|
||||
|
||||
headers = signer("HEAD", "/non-existent")
|
||||
response = client.head("/non-existent", headers=headers)
|
||||
assert response.status_code == 404
|
||||
|
||||
def test_head_object(client, signer):
|
||||
headers = signer("PUT", "/head-obj-test")
|
||||
client.put("/head-obj-test", headers=headers)
|
||||
|
||||
headers = signer("PUT", "/head-obj-test/obj", body=b"content")
|
||||
client.put("/head-obj-test/obj", headers=headers, data=b"content")
|
||||
|
||||
headers = signer("HEAD", "/head-obj-test/obj")
|
||||
response = client.head("/head-obj-test/obj", headers=headers)
|
||||
assert response.status_code == 200
|
||||
assert response.headers["ETag"]
|
||||
assert response.headers["Content-Length"] == "7"
|
||||
|
||||
headers = signer("HEAD", "/head-obj-test/missing")
|
||||
response = client.head("/head-obj-test/missing", headers=headers)
|
||||
assert response.status_code == 404
|
||||
|
||||
def test_list_parts(client, signer):
|
||||
# Create bucket
|
||||
headers = signer("PUT", "/multipart-test")
|
||||
client.put("/multipart-test", headers=headers)
|
||||
|
||||
# Initiate multipart upload
|
||||
headers = signer("POST", "/multipart-test/obj?uploads")
|
||||
response = client.post("/multipart-test/obj?uploads", headers=headers)
|
||||
assert response.status_code == 200
|
||||
from xml.etree.ElementTree import fromstring
|
||||
upload_id = fromstring(response.data).find("UploadId").text
|
||||
|
||||
# Upload part 1
|
||||
headers = signer("PUT", f"/multipart-test/obj?partNumber=1&uploadId={upload_id}", body=b"part1")
|
||||
client.put(f"/multipart-test/obj?partNumber=1&uploadId={upload_id}", headers=headers, data=b"part1")
|
||||
|
||||
# Upload part 2
|
||||
headers = signer("PUT", f"/multipart-test/obj?partNumber=2&uploadId={upload_id}", body=b"part2")
|
||||
client.put(f"/multipart-test/obj?partNumber=2&uploadId={upload_id}", headers=headers, data=b"part2")
|
||||
|
||||
# List parts
|
||||
headers = signer("GET", f"/multipart-test/obj?uploadId={upload_id}")
|
||||
response = client.get(f"/multipart-test/obj?uploadId={upload_id}", headers=headers)
|
||||
assert response.status_code == 200
|
||||
|
||||
root = fromstring(response.data)
|
||||
assert root.tag == "ListPartsResult"
|
||||
parts = root.findall("Part")
|
||||
assert len(parts) == 2
|
||||
assert parts[0].find("PartNumber").text == "1"
|
||||
assert parts[1].find("PartNumber").text == "2"
|
||||
460
tests/test_site_sync.py
Normal file
460
tests/test_site_sync.py
Normal file
@@ -0,0 +1,460 @@
|
||||
import io
|
||||
import json
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from app.connections import ConnectionStore, RemoteConnection
|
||||
from app.replication import (
|
||||
ReplicationManager,
|
||||
ReplicationRule,
|
||||
REPLICATION_MODE_BIDIRECTIONAL,
|
||||
REPLICATION_MODE_NEW_ONLY,
|
||||
)
|
||||
from app.site_sync import (
|
||||
SiteSyncWorker,
|
||||
SyncState,
|
||||
SyncedObjectInfo,
|
||||
SiteSyncStats,
|
||||
RemoteObjectMeta,
|
||||
)
|
||||
from app.storage import ObjectStorage
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def storage(tmp_path: Path):
|
||||
storage_root = tmp_path / "data"
|
||||
storage_root.mkdir(parents=True)
|
||||
return ObjectStorage(storage_root)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def connections(tmp_path: Path):
|
||||
connections_path = tmp_path / "connections.json"
|
||||
store = ConnectionStore(connections_path)
|
||||
conn = RemoteConnection(
|
||||
id="test-conn",
|
||||
name="Test Remote",
|
||||
endpoint_url="http://localhost:9000",
|
||||
access_key="remote-access",
|
||||
secret_key="remote-secret",
|
||||
region="us-east-1",
|
||||
)
|
||||
store.add(conn)
|
||||
return store
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def replication_manager(storage, connections, tmp_path):
|
||||
rules_path = tmp_path / "replication_rules.json"
|
||||
storage_root = tmp_path / "data"
|
||||
storage_root.mkdir(exist_ok=True)
|
||||
manager = ReplicationManager(storage, connections, rules_path, storage_root)
|
||||
yield manager
|
||||
manager.shutdown(wait=False)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def site_sync_worker(storage, connections, replication_manager, tmp_path):
|
||||
storage_root = tmp_path / "data"
|
||||
worker = SiteSyncWorker(
|
||||
storage=storage,
|
||||
connections=connections,
|
||||
replication_manager=replication_manager,
|
||||
storage_root=storage_root,
|
||||
interval_seconds=60,
|
||||
batch_size=100,
|
||||
)
|
||||
yield worker
|
||||
worker.shutdown()
|
||||
|
||||
|
||||
class TestSyncedObjectInfo:
|
||||
def test_to_dict(self):
|
||||
info = SyncedObjectInfo(
|
||||
last_synced_at=1234567890.0,
|
||||
remote_etag="abc123",
|
||||
source="remote",
|
||||
)
|
||||
result = info.to_dict()
|
||||
assert result["last_synced_at"] == 1234567890.0
|
||||
assert result["remote_etag"] == "abc123"
|
||||
assert result["source"] == "remote"
|
||||
|
||||
def test_from_dict(self):
|
||||
data = {
|
||||
"last_synced_at": 9876543210.0,
|
||||
"remote_etag": "def456",
|
||||
"source": "local",
|
||||
}
|
||||
info = SyncedObjectInfo.from_dict(data)
|
||||
assert info.last_synced_at == 9876543210.0
|
||||
assert info.remote_etag == "def456"
|
||||
assert info.source == "local"
|
||||
|
||||
|
||||
class TestSyncState:
|
||||
def test_to_dict(self):
|
||||
state = SyncState(
|
||||
synced_objects={
|
||||
"test.txt": SyncedObjectInfo(
|
||||
last_synced_at=1000.0,
|
||||
remote_etag="etag1",
|
||||
source="remote",
|
||||
)
|
||||
},
|
||||
last_full_sync=2000.0,
|
||||
)
|
||||
result = state.to_dict()
|
||||
assert "test.txt" in result["synced_objects"]
|
||||
assert result["synced_objects"]["test.txt"]["remote_etag"] == "etag1"
|
||||
assert result["last_full_sync"] == 2000.0
|
||||
|
||||
def test_from_dict(self):
|
||||
data = {
|
||||
"synced_objects": {
|
||||
"file.txt": {
|
||||
"last_synced_at": 3000.0,
|
||||
"remote_etag": "etag2",
|
||||
"source": "remote",
|
||||
}
|
||||
},
|
||||
"last_full_sync": 4000.0,
|
||||
}
|
||||
state = SyncState.from_dict(data)
|
||||
assert "file.txt" in state.synced_objects
|
||||
assert state.synced_objects["file.txt"].remote_etag == "etag2"
|
||||
assert state.last_full_sync == 4000.0
|
||||
|
||||
def test_from_dict_empty(self):
|
||||
state = SyncState.from_dict({})
|
||||
assert state.synced_objects == {}
|
||||
assert state.last_full_sync is None
|
||||
|
||||
|
||||
class TestSiteSyncStats:
|
||||
def test_to_dict(self):
|
||||
stats = SiteSyncStats(
|
||||
last_sync_at=1234567890.0,
|
||||
objects_pulled=10,
|
||||
objects_skipped=5,
|
||||
conflicts_resolved=2,
|
||||
deletions_applied=1,
|
||||
errors=0,
|
||||
)
|
||||
result = stats.to_dict()
|
||||
assert result["objects_pulled"] == 10
|
||||
assert result["objects_skipped"] == 5
|
||||
assert result["conflicts_resolved"] == 2
|
||||
assert result["deletions_applied"] == 1
|
||||
assert result["errors"] == 0
|
||||
|
||||
|
||||
class TestRemoteObjectMeta:
|
||||
def test_from_s3_object(self):
|
||||
obj = {
|
||||
"Key": "test/file.txt",
|
||||
"Size": 1024,
|
||||
"LastModified": datetime(2025, 1, 1, 12, 0, 0, tzinfo=timezone.utc),
|
||||
"ETag": '"abc123def456"',
|
||||
}
|
||||
meta = RemoteObjectMeta.from_s3_object(obj)
|
||||
assert meta.key == "test/file.txt"
|
||||
assert meta.size == 1024
|
||||
assert meta.last_modified == datetime(2025, 1, 1, 12, 0, 0, tzinfo=timezone.utc)
|
||||
assert meta.etag == "abc123def456"
|
||||
|
||||
|
||||
class TestReplicationRuleBidirectional:
|
||||
def test_rule_with_bidirectional_mode(self):
|
||||
rule = ReplicationRule(
|
||||
bucket_name="sync-bucket",
|
||||
target_connection_id="test-conn",
|
||||
target_bucket="remote-bucket",
|
||||
enabled=True,
|
||||
mode=REPLICATION_MODE_BIDIRECTIONAL,
|
||||
sync_deletions=True,
|
||||
)
|
||||
assert rule.mode == REPLICATION_MODE_BIDIRECTIONAL
|
||||
assert rule.sync_deletions is True
|
||||
assert rule.last_pull_at is None
|
||||
|
||||
def test_rule_to_dict_includes_new_fields(self):
|
||||
rule = ReplicationRule(
|
||||
bucket_name="sync-bucket",
|
||||
target_connection_id="test-conn",
|
||||
target_bucket="remote-bucket",
|
||||
mode=REPLICATION_MODE_BIDIRECTIONAL,
|
||||
sync_deletions=False,
|
||||
last_pull_at=1234567890.0,
|
||||
)
|
||||
result = rule.to_dict()
|
||||
assert result["mode"] == REPLICATION_MODE_BIDIRECTIONAL
|
||||
assert result["sync_deletions"] is False
|
||||
assert result["last_pull_at"] == 1234567890.0
|
||||
|
||||
def test_rule_from_dict_with_new_fields(self):
|
||||
data = {
|
||||
"bucket_name": "sync-bucket",
|
||||
"target_connection_id": "test-conn",
|
||||
"target_bucket": "remote-bucket",
|
||||
"mode": REPLICATION_MODE_BIDIRECTIONAL,
|
||||
"sync_deletions": False,
|
||||
"last_pull_at": 1234567890.0,
|
||||
}
|
||||
rule = ReplicationRule.from_dict(data)
|
||||
assert rule.mode == REPLICATION_MODE_BIDIRECTIONAL
|
||||
assert rule.sync_deletions is False
|
||||
assert rule.last_pull_at == 1234567890.0
|
||||
|
||||
def test_rule_from_dict_defaults_new_fields(self):
|
||||
data = {
|
||||
"bucket_name": "sync-bucket",
|
||||
"target_connection_id": "test-conn",
|
||||
"target_bucket": "remote-bucket",
|
||||
}
|
||||
rule = ReplicationRule.from_dict(data)
|
||||
assert rule.sync_deletions is True
|
||||
assert rule.last_pull_at is None
|
||||
|
||||
|
||||
class TestSiteSyncWorker:
|
||||
def test_start_and_shutdown(self, site_sync_worker):
|
||||
site_sync_worker.start()
|
||||
assert site_sync_worker._sync_thread is not None
|
||||
assert site_sync_worker._sync_thread.is_alive()
|
||||
site_sync_worker.shutdown()
|
||||
assert not site_sync_worker._sync_thread.is_alive()
|
||||
|
||||
def test_trigger_sync_no_rule(self, site_sync_worker):
|
||||
result = site_sync_worker.trigger_sync("nonexistent-bucket")
|
||||
assert result is None
|
||||
|
||||
def test_trigger_sync_wrong_mode(self, site_sync_worker, replication_manager):
|
||||
rule = ReplicationRule(
|
||||
bucket_name="new-only-bucket",
|
||||
target_connection_id="test-conn",
|
||||
target_bucket="remote-bucket",
|
||||
mode=REPLICATION_MODE_NEW_ONLY,
|
||||
enabled=True,
|
||||
)
|
||||
replication_manager.set_rule(rule)
|
||||
result = site_sync_worker.trigger_sync("new-only-bucket")
|
||||
assert result is None
|
||||
|
||||
def test_trigger_sync_disabled_rule(self, site_sync_worker, replication_manager):
|
||||
rule = ReplicationRule(
|
||||
bucket_name="disabled-bucket",
|
||||
target_connection_id="test-conn",
|
||||
target_bucket="remote-bucket",
|
||||
mode=REPLICATION_MODE_BIDIRECTIONAL,
|
||||
enabled=False,
|
||||
)
|
||||
replication_manager.set_rule(rule)
|
||||
result = site_sync_worker.trigger_sync("disabled-bucket")
|
||||
assert result is None
|
||||
|
||||
def test_get_stats_no_sync(self, site_sync_worker):
|
||||
stats = site_sync_worker.get_stats("nonexistent")
|
||||
assert stats is None
|
||||
|
||||
def test_resolve_conflict_remote_newer(self, site_sync_worker):
|
||||
local_meta = MagicMock()
|
||||
local_meta.last_modified = datetime(2025, 1, 1, 12, 0, 0, tzinfo=timezone.utc)
|
||||
local_meta.etag = "local123"
|
||||
|
||||
remote_meta = RemoteObjectMeta(
|
||||
key="test.txt",
|
||||
size=100,
|
||||
last_modified=datetime(2025, 1, 2, 12, 0, 0, tzinfo=timezone.utc),
|
||||
etag="remote456",
|
||||
)
|
||||
|
||||
result = site_sync_worker._resolve_conflict(local_meta, remote_meta)
|
||||
assert result == "pull"
|
||||
|
||||
def test_resolve_conflict_local_newer(self, site_sync_worker):
|
||||
local_meta = MagicMock()
|
||||
local_meta.last_modified = datetime(2025, 1, 2, 12, 0, 0, tzinfo=timezone.utc)
|
||||
local_meta.etag = "local123"
|
||||
|
||||
remote_meta = RemoteObjectMeta(
|
||||
key="test.txt",
|
||||
size=100,
|
||||
last_modified=datetime(2025, 1, 1, 12, 0, 0, tzinfo=timezone.utc),
|
||||
etag="remote456",
|
||||
)
|
||||
|
||||
result = site_sync_worker._resolve_conflict(local_meta, remote_meta)
|
||||
assert result == "keep"
|
||||
|
||||
def test_resolve_conflict_same_time_same_etag(self, site_sync_worker):
|
||||
ts = datetime(2025, 1, 1, 12, 0, 0, tzinfo=timezone.utc)
|
||||
local_meta = MagicMock()
|
||||
local_meta.last_modified = ts
|
||||
local_meta.etag = "same123"
|
||||
|
||||
remote_meta = RemoteObjectMeta(
|
||||
key="test.txt",
|
||||
size=100,
|
||||
last_modified=ts,
|
||||
etag="same123",
|
||||
)
|
||||
|
||||
result = site_sync_worker._resolve_conflict(local_meta, remote_meta)
|
||||
assert result == "skip"
|
||||
|
||||
def test_resolve_conflict_same_time_different_etag(self, site_sync_worker):
|
||||
ts = datetime(2025, 1, 1, 12, 0, 0, tzinfo=timezone.utc)
|
||||
local_meta = MagicMock()
|
||||
local_meta.last_modified = ts
|
||||
local_meta.etag = "aaa"
|
||||
|
||||
remote_meta = RemoteObjectMeta(
|
||||
key="test.txt",
|
||||
size=100,
|
||||
last_modified=ts,
|
||||
etag="zzz",
|
||||
)
|
||||
|
||||
result = site_sync_worker._resolve_conflict(local_meta, remote_meta)
|
||||
assert result == "pull"
|
||||
|
||||
def test_sync_state_persistence(self, site_sync_worker, tmp_path):
|
||||
bucket_name = "test-bucket"
|
||||
state = SyncState(
|
||||
synced_objects={
|
||||
"file1.txt": SyncedObjectInfo(
|
||||
last_synced_at=time.time(),
|
||||
remote_etag="etag1",
|
||||
source="remote",
|
||||
)
|
||||
},
|
||||
last_full_sync=time.time(),
|
||||
)
|
||||
|
||||
site_sync_worker._save_sync_state(bucket_name, state)
|
||||
|
||||
loaded = site_sync_worker._load_sync_state(bucket_name)
|
||||
assert "file1.txt" in loaded.synced_objects
|
||||
assert loaded.synced_objects["file1.txt"].remote_etag == "etag1"
|
||||
|
||||
def test_load_sync_state_nonexistent(self, site_sync_worker):
|
||||
state = site_sync_worker._load_sync_state("nonexistent-bucket")
|
||||
assert state.synced_objects == {}
|
||||
assert state.last_full_sync is None
|
||||
|
||||
@patch("app.site_sync._create_sync_client")
|
||||
def test_list_remote_objects(self, mock_create_client, site_sync_worker, connections, replication_manager):
|
||||
mock_client = MagicMock()
|
||||
mock_paginator = MagicMock()
|
||||
mock_paginator.paginate.return_value = [
|
||||
{
|
||||
"Contents": [
|
||||
{
|
||||
"Key": "file1.txt",
|
||||
"Size": 100,
|
||||
"LastModified": datetime(2025, 1, 1, tzinfo=timezone.utc),
|
||||
"ETag": '"etag1"',
|
||||
},
|
||||
{
|
||||
"Key": "file2.txt",
|
||||
"Size": 200,
|
||||
"LastModified": datetime(2025, 1, 2, tzinfo=timezone.utc),
|
||||
"ETag": '"etag2"',
|
||||
},
|
||||
]
|
||||
}
|
||||
]
|
||||
mock_client.get_paginator.return_value = mock_paginator
|
||||
mock_create_client.return_value = mock_client
|
||||
|
||||
rule = ReplicationRule(
|
||||
bucket_name="local-bucket",
|
||||
target_connection_id="test-conn",
|
||||
target_bucket="remote-bucket",
|
||||
mode=REPLICATION_MODE_BIDIRECTIONAL,
|
||||
)
|
||||
conn = connections.get("test-conn")
|
||||
|
||||
result = site_sync_worker._list_remote_objects(rule, conn)
|
||||
|
||||
assert "file1.txt" in result
|
||||
assert "file2.txt" in result
|
||||
assert result["file1.txt"].size == 100
|
||||
assert result["file2.txt"].size == 200
|
||||
|
||||
def test_list_local_objects(self, site_sync_worker, storage):
|
||||
storage.create_bucket("test-bucket")
|
||||
storage.put_object("test-bucket", "file1.txt", io.BytesIO(b"content1"))
|
||||
storage.put_object("test-bucket", "file2.txt", io.BytesIO(b"content2"))
|
||||
|
||||
result = site_sync_worker._list_local_objects("test-bucket")
|
||||
|
||||
assert "file1.txt" in result
|
||||
assert "file2.txt" in result
|
||||
|
||||
@patch("app.site_sync._create_sync_client")
|
||||
def test_sync_bucket_connection_not_found(self, mock_create_client, site_sync_worker, replication_manager):
|
||||
rule = ReplicationRule(
|
||||
bucket_name="test-bucket",
|
||||
target_connection_id="missing-conn",
|
||||
target_bucket="remote-bucket",
|
||||
mode=REPLICATION_MODE_BIDIRECTIONAL,
|
||||
enabled=True,
|
||||
)
|
||||
replication_manager.set_rule(rule)
|
||||
|
||||
stats = site_sync_worker._sync_bucket(rule)
|
||||
assert stats.errors == 1
|
||||
|
||||
|
||||
class TestSiteSyncIntegration:
|
||||
@patch("app.site_sync._create_sync_client")
|
||||
def test_full_sync_cycle(self, mock_create_client, site_sync_worker, storage, connections, replication_manager):
|
||||
storage.create_bucket("sync-bucket")
|
||||
storage.put_object("sync-bucket", "local-only.txt", io.BytesIO(b"local content"))
|
||||
|
||||
mock_client = MagicMock()
|
||||
mock_paginator = MagicMock()
|
||||
mock_paginator.paginate.return_value = [
|
||||
{
|
||||
"Contents": [
|
||||
{
|
||||
"Key": "remote-only.txt",
|
||||
"Size": 100,
|
||||
"LastModified": datetime(2025, 1, 15, tzinfo=timezone.utc),
|
||||
"ETag": '"remoteetag"',
|
||||
},
|
||||
]
|
||||
}
|
||||
]
|
||||
mock_client.get_paginator.return_value = mock_paginator
|
||||
mock_client.head_object.return_value = {"Metadata": {}}
|
||||
|
||||
def mock_download(bucket, key, path):
|
||||
Path(path).write_bytes(b"remote content")
|
||||
|
||||
mock_client.download_file.side_effect = mock_download
|
||||
mock_create_client.return_value = mock_client
|
||||
|
||||
rule = ReplicationRule(
|
||||
bucket_name="sync-bucket",
|
||||
target_connection_id="test-conn",
|
||||
target_bucket="remote-bucket",
|
||||
mode=REPLICATION_MODE_BIDIRECTIONAL,
|
||||
enabled=True,
|
||||
)
|
||||
replication_manager.set_rule(rule)
|
||||
|
||||
stats = site_sync_worker._sync_bucket(rule)
|
||||
|
||||
assert stats.objects_pulled == 1
|
||||
assert stats.errors == 0
|
||||
|
||||
objects = site_sync_worker._list_local_objects("sync-bucket")
|
||||
assert "local-only.txt" in objects
|
||||
assert "remote-only.txt" in objects
|
||||
@@ -1,8 +1,12 @@
|
||||
import io
|
||||
import json
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
from werkzeug.serving import make_server
|
||||
|
||||
from app import create_app
|
||||
from app.s3_client import S3ProxyClient
|
||||
|
||||
|
||||
def _build_app(tmp_path: Path):
|
||||
@@ -26,13 +30,32 @@ def _build_app(tmp_path: Path):
|
||||
"STORAGE_ROOT": storage_root,
|
||||
"IAM_CONFIG": iam_config,
|
||||
"BUCKET_POLICY_PATH": bucket_policies,
|
||||
"API_BASE_URL": "http://localhost",
|
||||
"API_BASE_URL": "http://127.0.0.1:0",
|
||||
"SECRET_KEY": "testing",
|
||||
"WTF_CSRF_ENABLED": False,
|
||||
}
|
||||
)
|
||||
|
||||
server = make_server("127.0.0.1", 0, app)
|
||||
host, port = server.server_address
|
||||
api_url = f"http://{host}:{port}"
|
||||
app.config["API_BASE_URL"] = api_url
|
||||
app.extensions["s3_proxy"] = S3ProxyClient(api_base_url=api_url)
|
||||
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
|
||||
app._test_server = server
|
||||
app._test_thread = thread
|
||||
return app
|
||||
|
||||
|
||||
def _shutdown_app(app):
|
||||
if hasattr(app, "_test_server"):
|
||||
app._test_server.shutdown()
|
||||
app._test_thread.join(timeout=2)
|
||||
|
||||
|
||||
def _login(client):
|
||||
return client.post(
|
||||
"/ui/login",
|
||||
@@ -43,54 +66,60 @@ def _login(client):
|
||||
|
||||
def test_bulk_delete_json_route(tmp_path: Path):
|
||||
app = _build_app(tmp_path)
|
||||
storage = app.extensions["object_storage"]
|
||||
storage.create_bucket("demo")
|
||||
storage.put_object("demo", "first.txt", io.BytesIO(b"first"))
|
||||
storage.put_object("demo", "second.txt", io.BytesIO(b"second"))
|
||||
try:
|
||||
storage = app.extensions["object_storage"]
|
||||
storage.create_bucket("demo")
|
||||
storage.put_object("demo", "first.txt", io.BytesIO(b"first"))
|
||||
storage.put_object("demo", "second.txt", io.BytesIO(b"second"))
|
||||
|
||||
client = app.test_client()
|
||||
assert _login(client).status_code == 200
|
||||
client = app.test_client()
|
||||
assert _login(client).status_code == 200
|
||||
|
||||
response = client.post(
|
||||
"/ui/buckets/demo/objects/bulk-delete",
|
||||
json={"keys": ["first.txt", "missing.txt"]},
|
||||
headers={"X-Requested-With": "XMLHttpRequest"},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
payload = response.get_json()
|
||||
assert payload["status"] == "ok"
|
||||
assert set(payload["deleted"]) == {"first.txt", "missing.txt"}
|
||||
assert payload["errors"] == []
|
||||
response = client.post(
|
||||
"/ui/buckets/demo/objects/bulk-delete",
|
||||
json={"keys": ["first.txt", "missing.txt"]},
|
||||
headers={"X-Requested-With": "XMLHttpRequest"},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
payload = response.get_json()
|
||||
assert payload["status"] == "ok"
|
||||
assert set(payload["deleted"]) == {"first.txt", "missing.txt"}
|
||||
assert payload["errors"] == []
|
||||
|
||||
listing = storage.list_objects_all("demo")
|
||||
assert {meta.key for meta in listing} == {"second.txt"}
|
||||
listing = storage.list_objects_all("demo")
|
||||
assert {meta.key for meta in listing} == {"second.txt"}
|
||||
finally:
|
||||
_shutdown_app(app)
|
||||
|
||||
|
||||
def test_bulk_delete_validation(tmp_path: Path):
|
||||
app = _build_app(tmp_path)
|
||||
storage = app.extensions["object_storage"]
|
||||
storage.create_bucket("demo")
|
||||
storage.put_object("demo", "keep.txt", io.BytesIO(b"keep"))
|
||||
try:
|
||||
storage = app.extensions["object_storage"]
|
||||
storage.create_bucket("demo")
|
||||
storage.put_object("demo", "keep.txt", io.BytesIO(b"keep"))
|
||||
|
||||
client = app.test_client()
|
||||
assert _login(client).status_code == 200
|
||||
client = app.test_client()
|
||||
assert _login(client).status_code == 200
|
||||
|
||||
bad_response = client.post(
|
||||
"/ui/buckets/demo/objects/bulk-delete",
|
||||
json={"keys": []},
|
||||
headers={"X-Requested-With": "XMLHttpRequest"},
|
||||
)
|
||||
assert bad_response.status_code == 400
|
||||
assert bad_response.get_json()["status"] == "error"
|
||||
bad_response = client.post(
|
||||
"/ui/buckets/demo/objects/bulk-delete",
|
||||
json={"keys": []},
|
||||
headers={"X-Requested-With": "XMLHttpRequest"},
|
||||
)
|
||||
assert bad_response.status_code == 400
|
||||
assert bad_response.get_json()["status"] == "error"
|
||||
|
||||
too_many = [f"obj-{index}.txt" for index in range(501)]
|
||||
limit_response = client.post(
|
||||
"/ui/buckets/demo/objects/bulk-delete",
|
||||
json={"keys": too_many},
|
||||
headers={"X-Requested-With": "XMLHttpRequest"},
|
||||
)
|
||||
assert limit_response.status_code == 400
|
||||
assert limit_response.get_json()["status"] == "error"
|
||||
too_many = [f"obj-{index}.txt" for index in range(501)]
|
||||
limit_response = client.post(
|
||||
"/ui/buckets/demo/objects/bulk-delete",
|
||||
json={"keys": too_many},
|
||||
headers={"X-Requested-With": "XMLHttpRequest"},
|
||||
)
|
||||
assert limit_response.status_code == 400
|
||||
assert limit_response.get_json()["status"] == "error"
|
||||
|
||||
still_there = storage.list_objects_all("demo")
|
||||
assert {meta.key for meta in still_there} == {"keep.txt"}
|
||||
still_there = storage.list_objects_all("demo")
|
||||
assert {meta.key for meta in still_there} == {"keep.txt"}
|
||||
finally:
|
||||
_shutdown_app(app)
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
"""Tests for UI-based encryption configuration."""
|
||||
import json
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from werkzeug.serving import make_server
|
||||
|
||||
from app import create_app
|
||||
from app.s3_client import S3ProxyClient
|
||||
|
||||
|
||||
def get_csrf_token(response):
|
||||
@@ -37,212 +40,224 @@ def _make_encryption_app(tmp_path: Path, *, kms_enabled: bool = True):
|
||||
]
|
||||
}
|
||||
iam_config.write_text(json.dumps(iam_payload))
|
||||
|
||||
|
||||
config = {
|
||||
"TESTING": True,
|
||||
"STORAGE_ROOT": storage_root,
|
||||
"IAM_CONFIG": iam_config,
|
||||
"BUCKET_POLICY_PATH": bucket_policies,
|
||||
"API_BASE_URL": "http://testserver",
|
||||
"API_BASE_URL": "http://127.0.0.1:0",
|
||||
"SECRET_KEY": "testing",
|
||||
"ENCRYPTION_ENABLED": True,
|
||||
"WTF_CSRF_ENABLED": False,
|
||||
}
|
||||
|
||||
|
||||
if kms_enabled:
|
||||
config["KMS_ENABLED"] = True
|
||||
config["KMS_KEYS_PATH"] = str(tmp_path / "kms_keys.json")
|
||||
config["ENCRYPTION_MASTER_KEY_PATH"] = str(tmp_path / "master.key")
|
||||
|
||||
|
||||
app = create_app(config)
|
||||
|
||||
server = make_server("127.0.0.1", 0, app)
|
||||
host, port = server.server_address
|
||||
api_url = f"http://{host}:{port}"
|
||||
app.config["API_BASE_URL"] = api_url
|
||||
app.extensions["s3_proxy"] = S3ProxyClient(api_base_url=api_url)
|
||||
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
|
||||
app._test_server = server
|
||||
app._test_thread = thread
|
||||
|
||||
storage = app.extensions["object_storage"]
|
||||
storage.create_bucket("test-bucket")
|
||||
return app
|
||||
|
||||
|
||||
def _shutdown_app(app):
|
||||
if hasattr(app, "_test_server"):
|
||||
app._test_server.shutdown()
|
||||
app._test_thread.join(timeout=2)
|
||||
|
||||
|
||||
class TestUIBucketEncryption:
|
||||
"""Test bucket encryption configuration via UI."""
|
||||
|
||||
|
||||
def test_bucket_detail_shows_encryption_card(self, tmp_path):
|
||||
"""Encryption card should be visible on bucket detail page."""
|
||||
app = _make_encryption_app(tmp_path)
|
||||
client = app.test_client()
|
||||
try:
|
||||
client = app.test_client()
|
||||
|
||||
client.post("/ui/login", data={"access_key": "test", "secret_key": "secret"}, follow_redirects=True)
|
||||
client.post("/ui/login", data={"access_key": "test", "secret_key": "secret"}, follow_redirects=True)
|
||||
|
||||
response = client.get("/ui/buckets/test-bucket?tab=properties")
|
||||
assert response.status_code == 200
|
||||
|
||||
html = response.data.decode("utf-8")
|
||||
assert "Default Encryption" in html
|
||||
assert "Encryption Algorithm" in html or "Default encryption disabled" in html
|
||||
finally:
|
||||
_shutdown_app(app)
|
||||
|
||||
response = client.get("/ui/buckets/test-bucket?tab=properties")
|
||||
assert response.status_code == 200
|
||||
|
||||
html = response.data.decode("utf-8")
|
||||
assert "Default Encryption" in html
|
||||
assert "Encryption Algorithm" in html or "Default encryption disabled" in html
|
||||
|
||||
def test_enable_aes256_encryption(self, tmp_path):
|
||||
"""Should be able to enable AES-256 encryption."""
|
||||
app = _make_encryption_app(tmp_path)
|
||||
client = app.test_client()
|
||||
try:
|
||||
client = app.test_client()
|
||||
|
||||
client.post("/ui/login", data={"access_key": "test", "secret_key": "secret"}, follow_redirects=True)
|
||||
client.post("/ui/login", data={"access_key": "test", "secret_key": "secret"}, follow_redirects=True)
|
||||
|
||||
response = client.get("/ui/buckets/test-bucket?tab=properties")
|
||||
csrf_token = get_csrf_token(response)
|
||||
response = client.post(
|
||||
"/ui/buckets/test-bucket/encryption",
|
||||
data={
|
||||
"action": "enable",
|
||||
"algorithm": "AES256",
|
||||
},
|
||||
follow_redirects=True,
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
html = response.data.decode("utf-8")
|
||||
assert "AES-256" in html or "encryption enabled" in html.lower()
|
||||
finally:
|
||||
_shutdown_app(app)
|
||||
|
||||
response = client.post(
|
||||
"/ui/buckets/test-bucket/encryption",
|
||||
data={
|
||||
"csrf_token": csrf_token,
|
||||
"action": "enable",
|
||||
"algorithm": "AES256",
|
||||
},
|
||||
follow_redirects=True,
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
html = response.data.decode("utf-8")
|
||||
assert "AES-256" in html or "encryption enabled" in html.lower()
|
||||
|
||||
def test_enable_kms_encryption(self, tmp_path):
|
||||
"""Should be able to enable KMS encryption."""
|
||||
app = _make_encryption_app(tmp_path, kms_enabled=True)
|
||||
client = app.test_client()
|
||||
try:
|
||||
with app.app_context():
|
||||
kms = app.extensions.get("kms")
|
||||
if kms:
|
||||
key = kms.create_key("test-key")
|
||||
key_id = key.key_id
|
||||
else:
|
||||
pytest.skip("KMS not available")
|
||||
|
||||
with app.app_context():
|
||||
kms = app.extensions.get("kms")
|
||||
if kms:
|
||||
key = kms.create_key("test-key")
|
||||
key_id = key.key_id
|
||||
else:
|
||||
pytest.skip("KMS not available")
|
||||
client = app.test_client()
|
||||
client.post("/ui/login", data={"access_key": "test", "secret_key": "secret"}, follow_redirects=True)
|
||||
|
||||
client.post("/ui/login", data={"access_key": "test", "secret_key": "secret"}, follow_redirects=True)
|
||||
response = client.post(
|
||||
"/ui/buckets/test-bucket/encryption",
|
||||
data={
|
||||
"action": "enable",
|
||||
"algorithm": "aws:kms",
|
||||
"kms_key_id": key_id,
|
||||
},
|
||||
follow_redirects=True,
|
||||
)
|
||||
|
||||
response = client.get("/ui/buckets/test-bucket?tab=properties")
|
||||
csrf_token = get_csrf_token(response)
|
||||
assert response.status_code == 200
|
||||
html = response.data.decode("utf-8")
|
||||
assert "KMS" in html or "encryption enabled" in html.lower()
|
||||
finally:
|
||||
_shutdown_app(app)
|
||||
|
||||
response = client.post(
|
||||
"/ui/buckets/test-bucket/encryption",
|
||||
data={
|
||||
"csrf_token": csrf_token,
|
||||
"action": "enable",
|
||||
"algorithm": "aws:kms",
|
||||
"kms_key_id": key_id,
|
||||
},
|
||||
follow_redirects=True,
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
html = response.data.decode("utf-8")
|
||||
assert "KMS" in html or "encryption enabled" in html.lower()
|
||||
|
||||
def test_disable_encryption(self, tmp_path):
|
||||
"""Should be able to disable encryption."""
|
||||
app = _make_encryption_app(tmp_path)
|
||||
client = app.test_client()
|
||||
try:
|
||||
client = app.test_client()
|
||||
|
||||
client.post("/ui/login", data={"access_key": "test", "secret_key": "secret"}, follow_redirects=True)
|
||||
client.post("/ui/login", data={"access_key": "test", "secret_key": "secret"}, follow_redirects=True)
|
||||
|
||||
response = client.get("/ui/buckets/test-bucket?tab=properties")
|
||||
csrf_token = get_csrf_token(response)
|
||||
|
||||
client.post(
|
||||
"/ui/buckets/test-bucket/encryption",
|
||||
data={
|
||||
"csrf_token": csrf_token,
|
||||
"action": "enable",
|
||||
"algorithm": "AES256",
|
||||
},
|
||||
)
|
||||
client.post(
|
||||
"/ui/buckets/test-bucket/encryption",
|
||||
data={
|
||||
"action": "enable",
|
||||
"algorithm": "AES256",
|
||||
},
|
||||
)
|
||||
|
||||
response = client.post(
|
||||
"/ui/buckets/test-bucket/encryption",
|
||||
data={
|
||||
"action": "disable",
|
||||
},
|
||||
follow_redirects=True,
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
html = response.data.decode("utf-8")
|
||||
assert "disabled" in html.lower() or "Default encryption disabled" in html
|
||||
finally:
|
||||
_shutdown_app(app)
|
||||
|
||||
response = client.get("/ui/buckets/test-bucket?tab=properties")
|
||||
csrf_token = get_csrf_token(response)
|
||||
|
||||
response = client.post(
|
||||
"/ui/buckets/test-bucket/encryption",
|
||||
data={
|
||||
"csrf_token": csrf_token,
|
||||
"action": "disable",
|
||||
},
|
||||
follow_redirects=True,
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
html = response.data.decode("utf-8")
|
||||
assert "disabled" in html.lower() or "Default encryption disabled" in html
|
||||
|
||||
def test_invalid_algorithm_rejected(self, tmp_path):
|
||||
"""Invalid encryption algorithm should be rejected."""
|
||||
app = _make_encryption_app(tmp_path)
|
||||
client = app.test_client()
|
||||
try:
|
||||
client = app.test_client()
|
||||
|
||||
client.post("/ui/login", data={"access_key": "test", "secret_key": "secret"}, follow_redirects=True)
|
||||
client.post("/ui/login", data={"access_key": "test", "secret_key": "secret"}, follow_redirects=True)
|
||||
|
||||
response = client.get("/ui/buckets/test-bucket?tab=properties")
|
||||
csrf_token = get_csrf_token(response)
|
||||
response = client.post(
|
||||
"/ui/buckets/test-bucket/encryption",
|
||||
data={
|
||||
"action": "enable",
|
||||
"algorithm": "INVALID",
|
||||
},
|
||||
follow_redirects=True,
|
||||
)
|
||||
|
||||
response = client.post(
|
||||
"/ui/buckets/test-bucket/encryption",
|
||||
data={
|
||||
"csrf_token": csrf_token,
|
||||
"action": "enable",
|
||||
"algorithm": "INVALID",
|
||||
},
|
||||
follow_redirects=True,
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
html = response.data.decode("utf-8")
|
||||
assert "Invalid" in html or "danger" in html
|
||||
assert response.status_code == 200
|
||||
html = response.data.decode("utf-8")
|
||||
assert "Invalid" in html or "danger" in html
|
||||
finally:
|
||||
_shutdown_app(app)
|
||||
|
||||
def test_encryption_persists_in_config(self, tmp_path):
|
||||
"""Encryption config should persist in bucket config."""
|
||||
app = _make_encryption_app(tmp_path)
|
||||
client = app.test_client()
|
||||
try:
|
||||
client = app.test_client()
|
||||
|
||||
client.post("/ui/login", data={"access_key": "test", "secret_key": "secret"}, follow_redirects=True)
|
||||
client.post("/ui/login", data={"access_key": "test", "secret_key": "secret"}, follow_redirects=True)
|
||||
|
||||
response = client.get("/ui/buckets/test-bucket?tab=properties")
|
||||
csrf_token = get_csrf_token(response)
|
||||
client.post(
|
||||
"/ui/buckets/test-bucket/encryption",
|
||||
data={
|
||||
"action": "enable",
|
||||
"algorithm": "AES256",
|
||||
},
|
||||
)
|
||||
|
||||
client.post(
|
||||
"/ui/buckets/test-bucket/encryption",
|
||||
data={
|
||||
"csrf_token": csrf_token,
|
||||
"action": "enable",
|
||||
"algorithm": "AES256",
|
||||
},
|
||||
)
|
||||
with app.app_context():
|
||||
storage = app.extensions["object_storage"]
|
||||
config = storage.get_bucket_encryption("test-bucket")
|
||||
|
||||
with app.app_context():
|
||||
storage = app.extensions["object_storage"]
|
||||
config = storage.get_bucket_encryption("test-bucket")
|
||||
|
||||
assert "Rules" in config
|
||||
assert len(config["Rules"]) == 1
|
||||
assert config["Rules"][0]["ApplyServerSideEncryptionByDefault"]["SSEAlgorithm"] == "AES256"
|
||||
assert "Rules" in config
|
||||
assert len(config["Rules"]) == 1
|
||||
assert config["Rules"][0]["SSEAlgorithm"] == "AES256"
|
||||
finally:
|
||||
_shutdown_app(app)
|
||||
|
||||
|
||||
class TestUIEncryptionWithoutPermission:
|
||||
"""Test encryption UI when user lacks permissions."""
|
||||
|
||||
|
||||
def test_readonly_user_cannot_change_encryption(self, tmp_path):
|
||||
"""Read-only user should not be able to change encryption settings."""
|
||||
app = _make_encryption_app(tmp_path)
|
||||
client = app.test_client()
|
||||
try:
|
||||
client = app.test_client()
|
||||
|
||||
client.post("/ui/login", data={"access_key": "readonly", "secret_key": "secret"}, follow_redirects=True)
|
||||
client.post("/ui/login", data={"access_key": "readonly", "secret_key": "secret"}, follow_redirects=True)
|
||||
|
||||
response = client.get("/ui/buckets/test-bucket?tab=properties")
|
||||
csrf_token = get_csrf_token(response)
|
||||
response = client.post(
|
||||
"/ui/buckets/test-bucket/encryption",
|
||||
data={
|
||||
"action": "enable",
|
||||
"algorithm": "AES256",
|
||||
},
|
||||
follow_redirects=True,
|
||||
)
|
||||
|
||||
response = client.post(
|
||||
"/ui/buckets/test-bucket/encryption",
|
||||
data={
|
||||
"csrf_token": csrf_token,
|
||||
"action": "enable",
|
||||
"algorithm": "AES256",
|
||||
},
|
||||
follow_redirects=True,
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
html = response.data.decode("utf-8")
|
||||
assert "Access denied" in html or "permission" in html.lower() or "not authorized" in html.lower()
|
||||
assert response.status_code == 200
|
||||
html = response.data.decode("utf-8")
|
||||
assert "Access denied" in html or "permission" in html.lower() or "not authorized" in html.lower()
|
||||
finally:
|
||||
_shutdown_app(app)
|
||||
|
||||
@@ -1,15 +1,18 @@
|
||||
"""Tests for UI pagination of bucket objects."""
|
||||
import json
|
||||
import threading
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from werkzeug.serving import make_server
|
||||
|
||||
from app import create_app
|
||||
from app.s3_client import S3ProxyClient
|
||||
|
||||
|
||||
def _make_app(tmp_path: Path):
|
||||
"""Create an app for testing."""
|
||||
"""Create an app for testing with a live API server."""
|
||||
storage_root = tmp_path / "data"
|
||||
iam_config = tmp_path / "iam.json"
|
||||
bucket_policies = tmp_path / "bucket_policies.json"
|
||||
@@ -28,161 +31,182 @@ def _make_app(tmp_path: Path):
|
||||
flask_app = create_app(
|
||||
{
|
||||
"TESTING": True,
|
||||
"SECRET_KEY": "testing",
|
||||
"WTF_CSRF_ENABLED": False,
|
||||
"STORAGE_ROOT": storage_root,
|
||||
"IAM_CONFIG": iam_config,
|
||||
"BUCKET_POLICY_PATH": bucket_policies,
|
||||
"API_BASE_URL": "http://127.0.0.1:0",
|
||||
}
|
||||
)
|
||||
|
||||
server = make_server("127.0.0.1", 0, flask_app)
|
||||
host, port = server.server_address
|
||||
api_url = f"http://{host}:{port}"
|
||||
flask_app.config["API_BASE_URL"] = api_url
|
||||
flask_app.extensions["s3_proxy"] = S3ProxyClient(api_base_url=api_url)
|
||||
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
|
||||
flask_app._test_server = server
|
||||
flask_app._test_thread = thread
|
||||
return flask_app
|
||||
|
||||
|
||||
def _shutdown_app(app):
|
||||
if hasattr(app, "_test_server"):
|
||||
app._test_server.shutdown()
|
||||
app._test_thread.join(timeout=2)
|
||||
|
||||
|
||||
class TestPaginatedObjectListing:
|
||||
"""Test paginated object listing API."""
|
||||
|
||||
def test_objects_api_returns_paginated_results(self, tmp_path):
|
||||
"""Objects API should return paginated results."""
|
||||
app = _make_app(tmp_path)
|
||||
storage = app.extensions["object_storage"]
|
||||
storage.create_bucket("test-bucket")
|
||||
|
||||
# Create 10 test objects
|
||||
for i in range(10):
|
||||
storage.put_object("test-bucket", f"file{i:02d}.txt", BytesIO(b"content"))
|
||||
|
||||
with app.test_client() as client:
|
||||
# Login first
|
||||
client.post("/ui/login", data={"access_key": "test", "secret_key": "secret"}, follow_redirects=True)
|
||||
|
||||
# Request first page of 3 objects
|
||||
resp = client.get("/ui/buckets/test-bucket/objects?max_keys=3")
|
||||
assert resp.status_code == 200
|
||||
|
||||
data = resp.get_json()
|
||||
assert len(data["objects"]) == 3
|
||||
assert data["is_truncated"] is True
|
||||
assert data["next_continuation_token"] is not None
|
||||
assert data["total_count"] == 10
|
||||
|
||||
try:
|
||||
storage = app.extensions["object_storage"]
|
||||
storage.create_bucket("test-bucket")
|
||||
|
||||
for i in range(10):
|
||||
storage.put_object("test-bucket", f"file{i:02d}.txt", BytesIO(b"content"))
|
||||
|
||||
with app.test_client() as client:
|
||||
client.post("/ui/login", data={"access_key": "test", "secret_key": "secret"}, follow_redirects=True)
|
||||
|
||||
resp = client.get("/ui/buckets/test-bucket/objects?max_keys=3")
|
||||
assert resp.status_code == 200
|
||||
|
||||
data = resp.get_json()
|
||||
assert len(data["objects"]) == 3
|
||||
assert data["is_truncated"] is True
|
||||
assert data["next_continuation_token"] is not None
|
||||
finally:
|
||||
_shutdown_app(app)
|
||||
|
||||
def test_objects_api_pagination_continuation(self, tmp_path):
|
||||
"""Objects API should support continuation tokens."""
|
||||
app = _make_app(tmp_path)
|
||||
storage = app.extensions["object_storage"]
|
||||
storage.create_bucket("test-bucket")
|
||||
|
||||
# Create 5 test objects
|
||||
for i in range(5):
|
||||
storage.put_object("test-bucket", f"file{i:02d}.txt", BytesIO(b"content"))
|
||||
|
||||
with app.test_client() as client:
|
||||
client.post("/ui/login", data={"access_key": "test", "secret_key": "secret"}, follow_redirects=True)
|
||||
|
||||
# Get first page
|
||||
resp = client.get("/ui/buckets/test-bucket/objects?max_keys=2")
|
||||
assert resp.status_code == 200
|
||||
data = resp.get_json()
|
||||
|
||||
first_page_keys = [obj["key"] for obj in data["objects"]]
|
||||
assert len(first_page_keys) == 2
|
||||
assert data["is_truncated"] is True
|
||||
|
||||
# Get second page
|
||||
token = data["next_continuation_token"]
|
||||
resp = client.get(f"/ui/buckets/test-bucket/objects?max_keys=2&continuation_token={token}")
|
||||
assert resp.status_code == 200
|
||||
data = resp.get_json()
|
||||
|
||||
second_page_keys = [obj["key"] for obj in data["objects"]]
|
||||
assert len(second_page_keys) == 2
|
||||
|
||||
# No overlap between pages
|
||||
assert set(first_page_keys).isdisjoint(set(second_page_keys))
|
||||
|
||||
try:
|
||||
storage = app.extensions["object_storage"]
|
||||
storage.create_bucket("test-bucket")
|
||||
|
||||
for i in range(5):
|
||||
storage.put_object("test-bucket", f"file{i:02d}.txt", BytesIO(b"content"))
|
||||
|
||||
with app.test_client() as client:
|
||||
client.post("/ui/login", data={"access_key": "test", "secret_key": "secret"}, follow_redirects=True)
|
||||
|
||||
resp = client.get("/ui/buckets/test-bucket/objects?max_keys=2")
|
||||
assert resp.status_code == 200
|
||||
data = resp.get_json()
|
||||
|
||||
first_page_keys = [obj["key"] for obj in data["objects"]]
|
||||
assert len(first_page_keys) == 2
|
||||
assert data["is_truncated"] is True
|
||||
|
||||
token = data["next_continuation_token"]
|
||||
resp = client.get(f"/ui/buckets/test-bucket/objects?max_keys=2&continuation_token={token}")
|
||||
assert resp.status_code == 200
|
||||
data = resp.get_json()
|
||||
|
||||
second_page_keys = [obj["key"] for obj in data["objects"]]
|
||||
assert len(second_page_keys) == 2
|
||||
|
||||
assert set(first_page_keys).isdisjoint(set(second_page_keys))
|
||||
finally:
|
||||
_shutdown_app(app)
|
||||
|
||||
def test_objects_api_prefix_filter(self, tmp_path):
|
||||
"""Objects API should support prefix filtering."""
|
||||
app = _make_app(tmp_path)
|
||||
storage = app.extensions["object_storage"]
|
||||
storage.create_bucket("test-bucket")
|
||||
|
||||
# Create objects with different prefixes
|
||||
storage.put_object("test-bucket", "logs/access.log", BytesIO(b"log"))
|
||||
storage.put_object("test-bucket", "logs/error.log", BytesIO(b"log"))
|
||||
storage.put_object("test-bucket", "data/file.txt", BytesIO(b"data"))
|
||||
|
||||
with app.test_client() as client:
|
||||
client.post("/ui/login", data={"access_key": "test", "secret_key": "secret"}, follow_redirects=True)
|
||||
|
||||
# Filter by prefix
|
||||
resp = client.get("/ui/buckets/test-bucket/objects?prefix=logs/")
|
||||
assert resp.status_code == 200
|
||||
data = resp.get_json()
|
||||
|
||||
keys = [obj["key"] for obj in data["objects"]]
|
||||
assert all(k.startswith("logs/") for k in keys)
|
||||
assert len(keys) == 2
|
||||
|
||||
try:
|
||||
storage = app.extensions["object_storage"]
|
||||
storage.create_bucket("test-bucket")
|
||||
|
||||
storage.put_object("test-bucket", "logs/access.log", BytesIO(b"log"))
|
||||
storage.put_object("test-bucket", "logs/error.log", BytesIO(b"log"))
|
||||
storage.put_object("test-bucket", "data/file.txt", BytesIO(b"data"))
|
||||
|
||||
with app.test_client() as client:
|
||||
client.post("/ui/login", data={"access_key": "test", "secret_key": "secret"}, follow_redirects=True)
|
||||
|
||||
resp = client.get("/ui/buckets/test-bucket/objects?prefix=logs/")
|
||||
assert resp.status_code == 200
|
||||
data = resp.get_json()
|
||||
|
||||
keys = [obj["key"] for obj in data["objects"]]
|
||||
assert all(k.startswith("logs/") for k in keys)
|
||||
assert len(keys) == 2
|
||||
finally:
|
||||
_shutdown_app(app)
|
||||
|
||||
def test_objects_api_requires_authentication(self, tmp_path):
|
||||
"""Objects API should require login."""
|
||||
app = _make_app(tmp_path)
|
||||
storage = app.extensions["object_storage"]
|
||||
storage.create_bucket("test-bucket")
|
||||
|
||||
with app.test_client() as client:
|
||||
# Don't login
|
||||
resp = client.get("/ui/buckets/test-bucket/objects")
|
||||
# Should redirect to login
|
||||
assert resp.status_code == 302
|
||||
assert "/ui/login" in resp.headers.get("Location", "")
|
||||
|
||||
try:
|
||||
storage = app.extensions["object_storage"]
|
||||
storage.create_bucket("test-bucket")
|
||||
|
||||
with app.test_client() as client:
|
||||
resp = client.get("/ui/buckets/test-bucket/objects")
|
||||
assert resp.status_code == 302
|
||||
assert "/ui/login" in resp.headers.get("Location", "")
|
||||
finally:
|
||||
_shutdown_app(app)
|
||||
|
||||
def test_objects_api_returns_object_metadata(self, tmp_path):
|
||||
"""Objects API should return complete object metadata."""
|
||||
app = _make_app(tmp_path)
|
||||
storage = app.extensions["object_storage"]
|
||||
storage.create_bucket("test-bucket")
|
||||
storage.put_object("test-bucket", "test.txt", BytesIO(b"test content"))
|
||||
|
||||
with app.test_client() as client:
|
||||
client.post("/ui/login", data={"access_key": "test", "secret_key": "secret"}, follow_redirects=True)
|
||||
|
||||
resp = client.get("/ui/buckets/test-bucket/objects")
|
||||
assert resp.status_code == 200
|
||||
data = resp.get_json()
|
||||
|
||||
assert len(data["objects"]) == 1
|
||||
obj = data["objects"][0]
|
||||
try:
|
||||
storage = app.extensions["object_storage"]
|
||||
storage.create_bucket("test-bucket")
|
||||
storage.put_object("test-bucket", "test.txt", BytesIO(b"test content"))
|
||||
|
||||
# Check all expected fields
|
||||
assert obj["key"] == "test.txt"
|
||||
assert obj["size"] == 12 # len("test content")
|
||||
assert "last_modified" in obj
|
||||
assert "last_modified_display" in obj
|
||||
assert "etag" in obj
|
||||
with app.test_client() as client:
|
||||
client.post("/ui/login", data={"access_key": "test", "secret_key": "secret"}, follow_redirects=True)
|
||||
|
||||
resp = client.get("/ui/buckets/test-bucket/objects")
|
||||
assert resp.status_code == 200
|
||||
data = resp.get_json()
|
||||
|
||||
assert len(data["objects"]) == 1
|
||||
obj = data["objects"][0]
|
||||
|
||||
assert obj["key"] == "test.txt"
|
||||
assert obj["size"] == 12
|
||||
assert "last_modified" in obj
|
||||
assert "last_modified_display" in obj
|
||||
assert "etag" in obj
|
||||
|
||||
assert "url_templates" in data
|
||||
templates = data["url_templates"]
|
||||
assert "preview" in templates
|
||||
assert "download" in templates
|
||||
assert "delete" in templates
|
||||
assert "KEY_PLACEHOLDER" in templates["preview"]
|
||||
finally:
|
||||
_shutdown_app(app)
|
||||
|
||||
# URLs are now returned as templates (not per-object) for performance
|
||||
assert "url_templates" in data
|
||||
templates = data["url_templates"]
|
||||
assert "preview" in templates
|
||||
assert "download" in templates
|
||||
assert "delete" in templates
|
||||
assert "KEY_PLACEHOLDER" in templates["preview"]
|
||||
|
||||
def test_bucket_detail_page_loads_without_objects(self, tmp_path):
|
||||
"""Bucket detail page should load even with many objects."""
|
||||
app = _make_app(tmp_path)
|
||||
storage = app.extensions["object_storage"]
|
||||
storage.create_bucket("test-bucket")
|
||||
|
||||
# Create many objects
|
||||
for i in range(100):
|
||||
storage.put_object("test-bucket", f"file{i:03d}.txt", BytesIO(b"x"))
|
||||
|
||||
with app.test_client() as client:
|
||||
client.post("/ui/login", data={"access_key": "test", "secret_key": "secret"}, follow_redirects=True)
|
||||
|
||||
# The page should load quickly (objects loaded via JS)
|
||||
resp = client.get("/ui/buckets/test-bucket")
|
||||
assert resp.status_code == 200
|
||||
|
||||
html = resp.data.decode("utf-8")
|
||||
# Should have the JavaScript loading infrastructure
|
||||
assert "loadObjects" in html or "objectsApiUrl" in html
|
||||
try:
|
||||
storage = app.extensions["object_storage"]
|
||||
storage.create_bucket("test-bucket")
|
||||
|
||||
for i in range(100):
|
||||
storage.put_object("test-bucket", f"file{i:03d}.txt", BytesIO(b"x"))
|
||||
|
||||
with app.test_client() as client:
|
||||
client.post("/ui/login", data={"access_key": "test", "secret_key": "secret"}, follow_redirects=True)
|
||||
|
||||
resp = client.get("/ui/buckets/test-bucket")
|
||||
assert resp.status_code == 200
|
||||
|
||||
html = resp.data.decode("utf-8")
|
||||
assert "bucket-detail-main.js" in html
|
||||
finally:
|
||||
_shutdown_app(app)
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
import io
|
||||
import json
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from werkzeug.serving import make_server
|
||||
|
||||
from app import create_app
|
||||
from app.s3_client import S3ProxyClient
|
||||
|
||||
|
||||
DENY_LIST_ALLOW_GET_POLICY = {
|
||||
@@ -47,11 +50,25 @@ def _make_ui_app(tmp_path: Path, *, enforce_policies: bool):
|
||||
"STORAGE_ROOT": storage_root,
|
||||
"IAM_CONFIG": iam_config,
|
||||
"BUCKET_POLICY_PATH": bucket_policies,
|
||||
"API_BASE_URL": "http://testserver",
|
||||
"API_BASE_URL": "http://127.0.0.1:0",
|
||||
"SECRET_KEY": "testing",
|
||||
"UI_ENFORCE_BUCKET_POLICIES": enforce_policies,
|
||||
"WTF_CSRF_ENABLED": False,
|
||||
}
|
||||
)
|
||||
|
||||
server = make_server("127.0.0.1", 0, app)
|
||||
host, port = server.server_address
|
||||
api_url = f"http://{host}:{port}"
|
||||
app.config["API_BASE_URL"] = api_url
|
||||
app.extensions["s3_proxy"] = S3ProxyClient(api_base_url=api_url)
|
||||
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
|
||||
app._test_server = server
|
||||
app._test_thread = thread
|
||||
|
||||
storage = app.extensions["object_storage"]
|
||||
storage.create_bucket("testbucket")
|
||||
storage.put_object("testbucket", "vid.mp4", io.BytesIO(b"video"))
|
||||
@@ -60,22 +77,28 @@ def _make_ui_app(tmp_path: Path, *, enforce_policies: bool):
|
||||
return app
|
||||
|
||||
|
||||
def _shutdown_app(app):
|
||||
if hasattr(app, "_test_server"):
|
||||
app._test_server.shutdown()
|
||||
app._test_thread.join(timeout=2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("enforce", [True, False])
|
||||
def test_ui_bucket_policy_enforcement_toggle(tmp_path: Path, enforce: bool):
|
||||
app = _make_ui_app(tmp_path, enforce_policies=enforce)
|
||||
client = app.test_client()
|
||||
client.post("/ui/login", data={"access_key": "test", "secret_key": "secret"}, follow_redirects=True)
|
||||
response = client.get("/ui/buckets/testbucket", follow_redirects=True)
|
||||
if enforce:
|
||||
assert b"Access denied by bucket policy" in response.data
|
||||
else:
|
||||
assert response.status_code == 200
|
||||
assert b"Access denied by bucket policy" not in response.data
|
||||
# Objects are now loaded via async API - check the objects endpoint
|
||||
objects_response = client.get("/ui/buckets/testbucket/objects")
|
||||
assert objects_response.status_code == 200
|
||||
data = objects_response.get_json()
|
||||
assert any(obj["key"] == "vid.mp4" for obj in data["objects"])
|
||||
try:
|
||||
client = app.test_client()
|
||||
client.post("/ui/login", data={"access_key": "test", "secret_key": "secret"}, follow_redirects=True)
|
||||
response = client.get("/ui/buckets/testbucket", follow_redirects=True)
|
||||
if enforce:
|
||||
assert b"Access denied by bucket policy" in response.data
|
||||
else:
|
||||
assert response.status_code == 200
|
||||
assert b"Access denied by bucket policy" not in response.data
|
||||
objects_response = client.get("/ui/buckets/testbucket/objects")
|
||||
assert objects_response.status_code == 403
|
||||
finally:
|
||||
_shutdown_app(app)
|
||||
|
||||
|
||||
def test_ui_bucket_policy_disabled_by_default(tmp_path: Path):
|
||||
@@ -99,23 +122,37 @@ def test_ui_bucket_policy_disabled_by_default(tmp_path: Path):
|
||||
"STORAGE_ROOT": storage_root,
|
||||
"IAM_CONFIG": iam_config,
|
||||
"BUCKET_POLICY_PATH": bucket_policies,
|
||||
"API_BASE_URL": "http://testserver",
|
||||
"API_BASE_URL": "http://127.0.0.1:0",
|
||||
"SECRET_KEY": "testing",
|
||||
"WTF_CSRF_ENABLED": False,
|
||||
}
|
||||
)
|
||||
storage = app.extensions["object_storage"]
|
||||
storage.create_bucket("testbucket")
|
||||
storage.put_object("testbucket", "vid.mp4", io.BytesIO(b"video"))
|
||||
policy_store = app.extensions["bucket_policies"]
|
||||
policy_store.set_policy("testbucket", DENY_LIST_ALLOW_GET_POLICY)
|
||||
|
||||
client = app.test_client()
|
||||
client.post("/ui/login", data={"access_key": "test", "secret_key": "secret"}, follow_redirects=True)
|
||||
response = client.get("/ui/buckets/testbucket", follow_redirects=True)
|
||||
assert response.status_code == 200
|
||||
assert b"Access denied by bucket policy" not in response.data
|
||||
# Objects are now loaded via async API - check the objects endpoint
|
||||
objects_response = client.get("/ui/buckets/testbucket/objects")
|
||||
assert objects_response.status_code == 200
|
||||
data = objects_response.get_json()
|
||||
assert any(obj["key"] == "vid.mp4" for obj in data["objects"])
|
||||
server = make_server("127.0.0.1", 0, app)
|
||||
host, port = server.server_address
|
||||
api_url = f"http://{host}:{port}"
|
||||
app.config["API_BASE_URL"] = api_url
|
||||
app.extensions["s3_proxy"] = S3ProxyClient(api_base_url=api_url)
|
||||
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
|
||||
app._test_server = server
|
||||
app._test_thread = thread
|
||||
|
||||
try:
|
||||
storage = app.extensions["object_storage"]
|
||||
storage.create_bucket("testbucket")
|
||||
storage.put_object("testbucket", "vid.mp4", io.BytesIO(b"video"))
|
||||
policy_store = app.extensions["bucket_policies"]
|
||||
policy_store.set_policy("testbucket", DENY_LIST_ALLOW_GET_POLICY)
|
||||
|
||||
client = app.test_client()
|
||||
client.post("/ui/login", data={"access_key": "test", "secret_key": "secret"}, follow_redirects=True)
|
||||
response = client.get("/ui/buckets/testbucket", follow_redirects=True)
|
||||
assert response.status_code == 200
|
||||
assert b"Access denied by bucket policy" not in response.data
|
||||
objects_response = client.get("/ui/buckets/testbucket/objects")
|
||||
assert objects_response.status_code == 403
|
||||
finally:
|
||||
_shutdown_app(app)
|
||||
|
||||
442
tests/test_website_hosting.py
Normal file
442
tests/test_website_hosting.py
Normal file
@@ -0,0 +1,442 @@
|
||||
import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
from xml.etree.ElementTree import fromstring
|
||||
|
||||
import pytest
|
||||
|
||||
from app import create_api_app
|
||||
from app.website_domains import WebsiteDomainStore
|
||||
|
||||
|
||||
def _stream(data: bytes):
|
||||
return io.BytesIO(data)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def website_app(tmp_path: Path):
|
||||
storage_root = tmp_path / "data"
|
||||
iam_config = tmp_path / "iam.json"
|
||||
bucket_policies = tmp_path / "bucket_policies.json"
|
||||
iam_payload = {
|
||||
"users": [
|
||||
{
|
||||
"access_key": "test",
|
||||
"secret_key": "secret",
|
||||
"display_name": "Test User",
|
||||
"policies": [{"bucket": "*", "actions": ["list", "read", "write", "delete", "policy", "iam:*"]}],
|
||||
}
|
||||
]
|
||||
}
|
||||
iam_config.write_text(json.dumps(iam_payload))
|
||||
flask_app = create_api_app(
|
||||
{
|
||||
"TESTING": True,
|
||||
"SECRET_KEY": "testing",
|
||||
"STORAGE_ROOT": storage_root,
|
||||
"IAM_CONFIG": iam_config,
|
||||
"BUCKET_POLICY_PATH": bucket_policies,
|
||||
"API_BASE_URL": "http://testserver",
|
||||
"WEBSITE_HOSTING_ENABLED": True,
|
||||
}
|
||||
)
|
||||
yield flask_app
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def website_client(website_app):
|
||||
return website_app.test_client()
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def storage(website_app):
|
||||
return website_app.extensions["object_storage"]
|
||||
|
||||
|
||||
class TestWebsiteDomainStore:
|
||||
def test_empty_store(self, tmp_path):
|
||||
store = WebsiteDomainStore(tmp_path / "domains.json")
|
||||
assert store.list_all() == []
|
||||
assert store.get_bucket("example.com") is None
|
||||
|
||||
def test_set_and_get_mapping(self, tmp_path):
|
||||
store = WebsiteDomainStore(tmp_path / "domains.json")
|
||||
store.set_mapping("example.com", "my-site")
|
||||
assert store.get_bucket("example.com") == "my-site"
|
||||
|
||||
def test_case_insensitive(self, tmp_path):
|
||||
store = WebsiteDomainStore(tmp_path / "domains.json")
|
||||
store.set_mapping("Example.COM", "my-site")
|
||||
assert store.get_bucket("example.com") == "my-site"
|
||||
assert store.get_bucket("EXAMPLE.COM") == "my-site"
|
||||
|
||||
def test_list_all(self, tmp_path):
|
||||
store = WebsiteDomainStore(tmp_path / "domains.json")
|
||||
store.set_mapping("a.com", "bucket-a")
|
||||
store.set_mapping("b.com", "bucket-b")
|
||||
result = store.list_all()
|
||||
domains = {item["domain"] for item in result}
|
||||
assert domains == {"a.com", "b.com"}
|
||||
|
||||
def test_delete_mapping(self, tmp_path):
|
||||
store = WebsiteDomainStore(tmp_path / "domains.json")
|
||||
store.set_mapping("example.com", "my-site")
|
||||
assert store.delete_mapping("example.com") is True
|
||||
assert store.get_bucket("example.com") is None
|
||||
|
||||
def test_delete_nonexistent(self, tmp_path):
|
||||
store = WebsiteDomainStore(tmp_path / "domains.json")
|
||||
assert store.delete_mapping("nope.com") is False
|
||||
|
||||
def test_overwrite_mapping(self, tmp_path):
|
||||
store = WebsiteDomainStore(tmp_path / "domains.json")
|
||||
store.set_mapping("example.com", "old-bucket")
|
||||
store.set_mapping("example.com", "new-bucket")
|
||||
assert store.get_bucket("example.com") == "new-bucket"
|
||||
|
||||
def test_persistence(self, tmp_path):
|
||||
path = tmp_path / "domains.json"
|
||||
store1 = WebsiteDomainStore(path)
|
||||
store1.set_mapping("example.com", "my-site")
|
||||
store2 = WebsiteDomainStore(path)
|
||||
assert store2.get_bucket("example.com") == "my-site"
|
||||
|
||||
def test_corrupt_file(self, tmp_path):
|
||||
path = tmp_path / "domains.json"
|
||||
path.write_text("not json")
|
||||
store = WebsiteDomainStore(path)
|
||||
assert store.list_all() == []
|
||||
|
||||
def test_non_dict_file(self, tmp_path):
|
||||
path = tmp_path / "domains.json"
|
||||
path.write_text('["not", "a", "dict"]')
|
||||
store = WebsiteDomainStore(path)
|
||||
assert store.list_all() == []
|
||||
|
||||
|
||||
class TestStorageWebsiteConfig:
|
||||
def test_get_website_no_config(self, storage):
|
||||
storage.create_bucket("test-bucket")
|
||||
assert storage.get_bucket_website("test-bucket") is None
|
||||
|
||||
def test_set_and_get_website(self, storage):
|
||||
storage.create_bucket("test-bucket")
|
||||
config = {"index_document": "index.html", "error_document": "error.html"}
|
||||
storage.set_bucket_website("test-bucket", config)
|
||||
result = storage.get_bucket_website("test-bucket")
|
||||
assert result["index_document"] == "index.html"
|
||||
assert result["error_document"] == "error.html"
|
||||
|
||||
def test_delete_website_config(self, storage):
|
||||
storage.create_bucket("test-bucket")
|
||||
storage.set_bucket_website("test-bucket", {"index_document": "index.html"})
|
||||
storage.set_bucket_website("test-bucket", None)
|
||||
assert storage.get_bucket_website("test-bucket") is None
|
||||
|
||||
def test_nonexistent_bucket(self, storage):
|
||||
with pytest.raises(Exception):
|
||||
storage.get_bucket_website("no-such-bucket")
|
||||
|
||||
|
||||
class TestS3WebsiteAPI:
|
||||
def test_put_website_config(self, website_client, signer):
|
||||
headers = signer("PUT", "/site-bucket")
|
||||
assert website_client.put("/site-bucket", headers=headers).status_code == 200
|
||||
|
||||
xml_body = b"""<WebsiteConfiguration>
|
||||
<IndexDocument><Suffix>index.html</Suffix></IndexDocument>
|
||||
<ErrorDocument><Key>404.html</Key></ErrorDocument>
|
||||
</WebsiteConfiguration>"""
|
||||
headers = signer("PUT", "/site-bucket?website",
|
||||
headers={"Content-Type": "application/xml"}, body=xml_body)
|
||||
resp = website_client.put("/site-bucket", query_string={"website": ""},
|
||||
headers=headers, data=xml_body, content_type="application/xml")
|
||||
assert resp.status_code == 200
|
||||
|
||||
def test_get_website_config(self, website_client, signer, storage):
|
||||
storage.create_bucket("site-bucket")
|
||||
storage.set_bucket_website("site-bucket", {
|
||||
"index_document": "index.html",
|
||||
"error_document": "error.html",
|
||||
})
|
||||
|
||||
headers = signer("GET", "/site-bucket?website")
|
||||
resp = website_client.get("/site-bucket", query_string={"website": ""}, headers=headers)
|
||||
assert resp.status_code == 200
|
||||
|
||||
root = fromstring(resp.data)
|
||||
suffix = root.find(".//{http://s3.amazonaws.com/doc/2006-03-01/}Suffix")
|
||||
if suffix is None:
|
||||
suffix = root.find(".//Suffix")
|
||||
assert suffix is not None
|
||||
assert suffix.text == "index.html"
|
||||
|
||||
def test_get_website_config_not_set(self, website_client, signer, storage):
|
||||
storage.create_bucket("no-website")
|
||||
headers = signer("GET", "/no-website?website")
|
||||
resp = website_client.get("/no-website", query_string={"website": ""}, headers=headers)
|
||||
assert resp.status_code == 404
|
||||
|
||||
def test_delete_website_config(self, website_client, signer, storage):
|
||||
storage.create_bucket("site-bucket")
|
||||
storage.set_bucket_website("site-bucket", {"index_document": "index.html"})
|
||||
|
||||
headers = signer("DELETE", "/site-bucket?website")
|
||||
resp = website_client.delete("/site-bucket", query_string={"website": ""}, headers=headers)
|
||||
assert resp.status_code == 204
|
||||
assert storage.get_bucket_website("site-bucket") is None
|
||||
|
||||
def test_put_website_missing_index(self, website_client, signer, storage):
|
||||
storage.create_bucket("site-bucket")
|
||||
xml_body = b"""<WebsiteConfiguration>
|
||||
<ErrorDocument><Key>error.html</Key></ErrorDocument>
|
||||
</WebsiteConfiguration>"""
|
||||
headers = signer("PUT", "/site-bucket?website",
|
||||
headers={"Content-Type": "application/xml"}, body=xml_body)
|
||||
resp = website_client.put("/site-bucket", query_string={"website": ""},
|
||||
headers=headers, data=xml_body, content_type="application/xml")
|
||||
assert resp.status_code == 400
|
||||
|
||||
def test_put_website_slash_in_suffix(self, website_client, signer, storage):
|
||||
storage.create_bucket("site-bucket")
|
||||
xml_body = b"""<WebsiteConfiguration>
|
||||
<IndexDocument><Suffix>path/index.html</Suffix></IndexDocument>
|
||||
</WebsiteConfiguration>"""
|
||||
headers = signer("PUT", "/site-bucket?website",
|
||||
headers={"Content-Type": "application/xml"}, body=xml_body)
|
||||
resp = website_client.put("/site-bucket", query_string={"website": ""},
|
||||
headers=headers, data=xml_body, content_type="application/xml")
|
||||
assert resp.status_code == 400
|
||||
|
||||
def test_put_website_malformed_xml(self, website_client, signer, storage):
|
||||
storage.create_bucket("site-bucket")
|
||||
xml_body = b"not xml at all"
|
||||
headers = signer("PUT", "/site-bucket?website",
|
||||
headers={"Content-Type": "application/xml"}, body=xml_body)
|
||||
resp = website_client.put("/site-bucket", query_string={"website": ""},
|
||||
headers=headers, data=xml_body, content_type="application/xml")
|
||||
assert resp.status_code == 400
|
||||
|
||||
def test_website_disabled(self, client, signer):
|
||||
headers = signer("PUT", "/test-bucket")
|
||||
assert client.put("/test-bucket", headers=headers).status_code == 200
|
||||
headers = signer("GET", "/test-bucket?website")
|
||||
resp = client.get("/test-bucket", query_string={"website": ""}, headers=headers)
|
||||
assert resp.status_code == 400
|
||||
assert b"not enabled" in resp.data
|
||||
|
||||
|
||||
class TestAdminWebsiteDomains:
|
||||
def _admin_headers(self, signer):
|
||||
return signer("GET", "/admin/website-domains")
|
||||
|
||||
def test_list_empty(self, website_client, signer):
|
||||
headers = self._admin_headers(signer)
|
||||
resp = website_client.get("/admin/website-domains", headers=headers)
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json() == []
|
||||
|
||||
def test_create_mapping(self, website_client, signer, storage):
|
||||
storage.create_bucket("my-site")
|
||||
headers = signer("POST", "/admin/website-domains",
|
||||
headers={"Content-Type": "application/json"},
|
||||
body=json.dumps({"domain": "example.com", "bucket": "my-site"}).encode())
|
||||
resp = website_client.post("/admin/website-domains",
|
||||
headers=headers,
|
||||
json={"domain": "example.com", "bucket": "my-site"})
|
||||
assert resp.status_code == 201
|
||||
data = resp.get_json()
|
||||
assert data["domain"] == "example.com"
|
||||
assert data["bucket"] == "my-site"
|
||||
|
||||
def test_create_duplicate(self, website_client, signer, storage):
|
||||
storage.create_bucket("my-site")
|
||||
body = json.dumps({"domain": "dup.com", "bucket": "my-site"}).encode()
|
||||
headers = signer("POST", "/admin/website-domains",
|
||||
headers={"Content-Type": "application/json"}, body=body)
|
||||
website_client.post("/admin/website-domains", headers=headers,
|
||||
json={"domain": "dup.com", "bucket": "my-site"})
|
||||
headers = signer("POST", "/admin/website-domains",
|
||||
headers={"Content-Type": "application/json"}, body=body)
|
||||
resp = website_client.post("/admin/website-domains", headers=headers,
|
||||
json={"domain": "dup.com", "bucket": "my-site"})
|
||||
assert resp.status_code == 409
|
||||
|
||||
def test_create_missing_domain(self, website_client, signer, storage):
|
||||
storage.create_bucket("my-site")
|
||||
body = json.dumps({"bucket": "my-site"}).encode()
|
||||
headers = signer("POST", "/admin/website-domains",
|
||||
headers={"Content-Type": "application/json"}, body=body)
|
||||
resp = website_client.post("/admin/website-domains", headers=headers,
|
||||
json={"bucket": "my-site"})
|
||||
assert resp.status_code == 400
|
||||
|
||||
def test_create_nonexistent_bucket(self, website_client, signer):
|
||||
body = json.dumps({"domain": "x.com", "bucket": "no-such"}).encode()
|
||||
headers = signer("POST", "/admin/website-domains",
|
||||
headers={"Content-Type": "application/json"}, body=body)
|
||||
resp = website_client.post("/admin/website-domains", headers=headers,
|
||||
json={"domain": "x.com", "bucket": "no-such"})
|
||||
assert resp.status_code == 404
|
||||
|
||||
def test_get_mapping(self, website_client, signer, storage):
|
||||
storage.create_bucket("my-site")
|
||||
body = json.dumps({"domain": "get.com", "bucket": "my-site"}).encode()
|
||||
headers = signer("POST", "/admin/website-domains",
|
||||
headers={"Content-Type": "application/json"}, body=body)
|
||||
website_client.post("/admin/website-domains", headers=headers,
|
||||
json={"domain": "get.com", "bucket": "my-site"})
|
||||
|
||||
headers = signer("GET", "/admin/website-domains/get.com")
|
||||
resp = website_client.get("/admin/website-domains/get.com", headers=headers)
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json()["bucket"] == "my-site"
|
||||
|
||||
def test_get_nonexistent(self, website_client, signer):
|
||||
headers = signer("GET", "/admin/website-domains/nope.com")
|
||||
resp = website_client.get("/admin/website-domains/nope.com", headers=headers)
|
||||
assert resp.status_code == 404
|
||||
|
||||
def test_update_mapping(self, website_client, signer, storage):
|
||||
storage.create_bucket("old-bucket")
|
||||
storage.create_bucket("new-bucket")
|
||||
body = json.dumps({"domain": "upd.com", "bucket": "old-bucket"}).encode()
|
||||
headers = signer("POST", "/admin/website-domains",
|
||||
headers={"Content-Type": "application/json"}, body=body)
|
||||
website_client.post("/admin/website-domains", headers=headers,
|
||||
json={"domain": "upd.com", "bucket": "old-bucket"})
|
||||
|
||||
body = json.dumps({"bucket": "new-bucket"}).encode()
|
||||
headers = signer("PUT", "/admin/website-domains/upd.com",
|
||||
headers={"Content-Type": "application/json"}, body=body)
|
||||
resp = website_client.put("/admin/website-domains/upd.com", headers=headers,
|
||||
json={"bucket": "new-bucket"})
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json()["bucket"] == "new-bucket"
|
||||
|
||||
def test_delete_mapping(self, website_client, signer, storage):
|
||||
storage.create_bucket("del-bucket")
|
||||
body = json.dumps({"domain": "del.com", "bucket": "del-bucket"}).encode()
|
||||
headers = signer("POST", "/admin/website-domains",
|
||||
headers={"Content-Type": "application/json"}, body=body)
|
||||
website_client.post("/admin/website-domains", headers=headers,
|
||||
json={"domain": "del.com", "bucket": "del-bucket"})
|
||||
|
||||
headers = signer("DELETE", "/admin/website-domains/del.com")
|
||||
resp = website_client.delete("/admin/website-domains/del.com", headers=headers)
|
||||
assert resp.status_code == 204
|
||||
|
||||
def test_delete_nonexistent(self, website_client, signer):
|
||||
headers = signer("DELETE", "/admin/website-domains/nope.com")
|
||||
resp = website_client.delete("/admin/website-domains/nope.com", headers=headers)
|
||||
assert resp.status_code == 404
|
||||
|
||||
def test_disabled(self, website_client, signer):
|
||||
with website_client.application.test_request_context():
|
||||
website_client.application.config["WEBSITE_HOSTING_ENABLED"] = False
|
||||
headers = signer("GET", "/admin/website-domains")
|
||||
resp = website_client.get("/admin/website-domains", headers=headers)
|
||||
assert resp.status_code == 400
|
||||
website_client.application.config["WEBSITE_HOSTING_ENABLED"] = True
|
||||
|
||||
|
||||
class TestWebsiteServing:
|
||||
def _setup_website(self, storage, website_app):
|
||||
storage.create_bucket("my-site")
|
||||
storage.put_object("my-site", "index.html", _stream(b"<h1>Home</h1>"))
|
||||
storage.put_object("my-site", "about.html", _stream(b"<h1>About</h1>"))
|
||||
storage.put_object("my-site", "assets/style.css", _stream(b"body { color: red; }"))
|
||||
storage.put_object("my-site", "sub/index.html", _stream(b"<h1>Sub</h1>"))
|
||||
storage.put_object("my-site", "404.html", _stream(b"<h1>Not Found</h1>"))
|
||||
storage.set_bucket_website("my-site", {
|
||||
"index_document": "index.html",
|
||||
"error_document": "404.html",
|
||||
})
|
||||
store = website_app.extensions["website_domains"]
|
||||
store.set_mapping("mysite.example.com", "my-site")
|
||||
|
||||
def test_serve_index(self, website_client, storage, website_app):
|
||||
self._setup_website(storage, website_app)
|
||||
resp = website_client.get("/", headers={"Host": "mysite.example.com"})
|
||||
assert resp.status_code == 200
|
||||
assert b"<h1>Home</h1>" in resp.data
|
||||
assert "text/html" in resp.content_type
|
||||
|
||||
def test_serve_specific_file(self, website_client, storage, website_app):
|
||||
self._setup_website(storage, website_app)
|
||||
resp = website_client.get("/about.html", headers={"Host": "mysite.example.com"})
|
||||
assert resp.status_code == 200
|
||||
assert b"<h1>About</h1>" in resp.data
|
||||
|
||||
def test_serve_css(self, website_client, storage, website_app):
|
||||
self._setup_website(storage, website_app)
|
||||
resp = website_client.get("/assets/style.css", headers={"Host": "mysite.example.com"})
|
||||
assert resp.status_code == 200
|
||||
assert b"body { color: red; }" in resp.data
|
||||
assert "text/css" in resp.content_type
|
||||
|
||||
def test_serve_subdirectory_index(self, website_client, storage, website_app):
|
||||
self._setup_website(storage, website_app)
|
||||
resp = website_client.get("/sub/", headers={"Host": "mysite.example.com"})
|
||||
assert resp.status_code == 200
|
||||
assert b"<h1>Sub</h1>" in resp.data
|
||||
|
||||
def test_serve_subdirectory_no_trailing_slash(self, website_client, storage, website_app):
|
||||
self._setup_website(storage, website_app)
|
||||
resp = website_client.get("/sub", headers={"Host": "mysite.example.com"})
|
||||
assert resp.status_code == 200
|
||||
assert b"<h1>Sub</h1>" in resp.data
|
||||
|
||||
def test_serve_error_document(self, website_client, storage, website_app):
|
||||
self._setup_website(storage, website_app)
|
||||
resp = website_client.get("/nonexistent.html", headers={"Host": "mysite.example.com"})
|
||||
assert resp.status_code == 404
|
||||
assert b"<h1>Not Found</h1>" in resp.data
|
||||
|
||||
def test_unmapped_host_passes_through(self, website_client, storage, website_app):
|
||||
self._setup_website(storage, website_app)
|
||||
resp = website_client.get("/", headers={"Host": "unknown.example.com"})
|
||||
assert resp.status_code != 200 or b"<h1>Home</h1>" not in resp.data
|
||||
|
||||
def test_head_request(self, website_client, storage, website_app):
|
||||
self._setup_website(storage, website_app)
|
||||
resp = website_client.head("/index.html", headers={"Host": "mysite.example.com"})
|
||||
assert resp.status_code == 200
|
||||
assert "Content-Length" in resp.headers
|
||||
assert resp.data == b""
|
||||
|
||||
def test_post_not_intercepted(self, website_client, storage, website_app):
|
||||
self._setup_website(storage, website_app)
|
||||
resp = website_client.post("/index.html", headers={"Host": "mysite.example.com"})
|
||||
assert resp.status_code != 200 or b"<h1>Home</h1>" not in resp.data
|
||||
|
||||
def test_bucket_deleted(self, website_client, storage, website_app):
|
||||
self._setup_website(storage, website_app)
|
||||
for obj in storage.list_objects_all("my-site"):
|
||||
storage.delete_object("my-site", obj.key)
|
||||
storage.delete_bucket("my-site")
|
||||
resp = website_client.get("/", headers={"Host": "mysite.example.com"})
|
||||
assert resp.status_code == 404
|
||||
|
||||
def test_no_website_config(self, website_client, storage, website_app):
|
||||
storage.create_bucket("bare-bucket")
|
||||
store = website_app.extensions["website_domains"]
|
||||
store.set_mapping("bare.example.com", "bare-bucket")
|
||||
resp = website_client.get("/", headers={"Host": "bare.example.com"})
|
||||
assert resp.status_code == 404
|
||||
|
||||
def test_host_with_port(self, website_client, storage, website_app):
|
||||
self._setup_website(storage, website_app)
|
||||
resp = website_client.get("/", headers={"Host": "mysite.example.com:5000"})
|
||||
assert resp.status_code == 200
|
||||
assert b"<h1>Home</h1>" in resp.data
|
||||
|
||||
def test_no_error_document(self, website_client, storage, website_app):
|
||||
storage.create_bucket("no-err")
|
||||
storage.put_object("no-err", "index.html", _stream(b"<h1>Home</h1>"))
|
||||
storage.set_bucket_website("no-err", {"index_document": "index.html"})
|
||||
store = website_app.extensions["website_domains"]
|
||||
store.set_mapping("noerr.example.com", "no-err")
|
||||
resp = website_client.get("/missing.html", headers={"Host": "noerr.example.com"})
|
||||
assert resp.status_code == 404
|
||||
assert b"Not Found" in resp.data
|
||||
Reference in New Issue
Block a user