Merge branch 'refactor/backend-architecture' into development
This commit is contained in:
yangshilin
204
api/app/features/memoir/memoir_images/storage.py
Normal file
204
api/app/features/memoir/memoir_images/storage.py
Normal file
@@ -0,0 +1,204 @@
|
||||
from app.core.logging import get_logger
|
||||
from urllib.parse import urlparse, urlunparse
|
||||
|
||||
from qcloud_cos import CosConfig, CosS3Client
|
||||
from qcloud_cos.cos_exception import CosClientError, CosServiceError
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
def normalize_cos_base_url(base_url: str, bucket: str, region: str) -> str:
|
||||
candidate = (base_url or "").rstrip("/")
|
||||
if not candidate and bucket and region:
|
||||
candidate = f"https://{bucket}.cos.{region}.myqcloud.com"
|
||||
if not candidate:
|
||||
return ""
|
||||
|
||||
parsed = urlparse(candidate)
|
||||
duplicated_appid_host = f"{bucket}-appid.cos." if bucket else ""
|
||||
if duplicated_appid_host and parsed.netloc.startswith(duplicated_appid_host):
|
||||
parsed = parsed._replace(
|
||||
netloc=parsed.netloc.replace(duplicated_appid_host, f"{bucket}.cos.", 1),
|
||||
path=parsed.path.rstrip("/"),
|
||||
)
|
||||
return urlunparse(parsed).rstrip("/")
|
||||
|
||||
return candidate
|
||||
|
||||
|
||||
def normalize_cos_url(url: str | None, bucket: str, region: str, base_url: str | None = None) -> str | None:
|
||||
if not url:
|
||||
return url
|
||||
|
||||
parsed = urlparse(url)
|
||||
if not parsed.scheme or not parsed.netloc:
|
||||
return url
|
||||
|
||||
normalized_base = normalize_cos_base_url(
|
||||
base_url or f"{parsed.scheme}://{parsed.netloc}",
|
||||
bucket=bucket,
|
||||
region=region,
|
||||
)
|
||||
if not normalized_base:
|
||||
return url
|
||||
|
||||
normalized_parsed = urlparse(normalized_base)
|
||||
return urlunparse(parsed._replace(scheme=normalized_parsed.scheme, netloc=normalized_parsed.netloc))
|
||||
|
||||
|
||||
def resolve_image_storage_key(image: dict | None) -> str | None:
|
||||
if not image:
|
||||
return None
|
||||
|
||||
explicit_key = image.get("storage_key")
|
||||
if explicit_key:
|
||||
return explicit_key
|
||||
|
||||
url = image.get("url")
|
||||
if not url:
|
||||
return None
|
||||
|
||||
parsed = urlparse(url)
|
||||
if parsed.scheme and parsed.netloc:
|
||||
key = parsed.path.lstrip("/")
|
||||
return key or None
|
||||
|
||||
return str(url).lstrip("/") or None
|
||||
|
||||
|
||||
def mark_image_delivery_unavailable(
|
||||
image: dict | None,
|
||||
*,
|
||||
error_message: str = "image delivery unavailable",
|
||||
) -> dict:
|
||||
normalized = dict(image or {})
|
||||
normalized["url"] = None
|
||||
normalized["error"] = normalized.get("error") or error_message
|
||||
return normalized
|
||||
|
||||
|
||||
class CosStorageError(Exception):
|
||||
"""Base exception for COS storage operations."""
|
||||
|
||||
def __init__(self, message: str, *, retryable: bool = True, request_id: str = ""):
|
||||
super().__init__(message)
|
||||
self.retryable = retryable
|
||||
self.request_id = request_id
|
||||
|
||||
|
||||
class CosUploadError(CosStorageError):
|
||||
"""Raised when put_object fails."""
|
||||
|
||||
|
||||
class CosDownloadUrlError(CosStorageError):
|
||||
"""Raised when generating a presigned download URL fails."""
|
||||
|
||||
|
||||
def _is_retryable_cos_error(exc: Exception) -> bool:
|
||||
if isinstance(exc, CosClientError):
|
||||
return True
|
||||
if isinstance(exc, CosServiceError):
|
||||
try:
|
||||
return exc.get_status_code() >= 500
|
||||
except Exception:
|
||||
return True
|
||||
return True
|
||||
|
||||
|
||||
class TencentCosStorageService:
|
||||
_instance: "TencentCosStorageService | None" = None
|
||||
_instance_config: tuple[str, str, str, str, str, str] | None = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
secret_id: str,
|
||||
secret_key: str,
|
||||
region: str,
|
||||
bucket: str,
|
||||
base_url: str,
|
||||
token: str = "",
|
||||
):
|
||||
self.secret_id = secret_id
|
||||
self.secret_key = secret_key
|
||||
self.bucket = bucket
|
||||
self.region = region
|
||||
self.base_url = normalize_cos_base_url(base_url, bucket=bucket, region=region)
|
||||
config = CosConfig(
|
||||
Region=region,
|
||||
SecretId=secret_id,
|
||||
SecretKey=secret_key,
|
||||
Token=token,
|
||||
Scheme="https",
|
||||
)
|
||||
self.client = CosS3Client(config)
|
||||
|
||||
def upload_bytes(self, image_bytes: bytes, key: str, content_type: str) -> str:
|
||||
try:
|
||||
response = self.client.put_object(
|
||||
Bucket=self.bucket,
|
||||
Body=image_bytes,
|
||||
Key=key,
|
||||
ContentType=content_type,
|
||||
)
|
||||
etag = response.get("ETag", "")
|
||||
request_id = response.get("x-cos-request-id", "")
|
||||
logger.info(
|
||||
"COS upload ok: key=%s, ETag=%s, request_id=%s", key, etag, request_id
|
||||
)
|
||||
except (CosClientError, CosServiceError) as exc:
|
||||
retryable = _is_retryable_cos_error(exc)
|
||||
request_id = (
|
||||
exc.get_request_id() if isinstance(exc, CosServiceError) else ""
|
||||
)
|
||||
raise CosUploadError(
|
||||
f"COS upload failed: key={key}, error={exc}",
|
||||
retryable=retryable,
|
||||
request_id=request_id,
|
||||
) from exc
|
||||
return f"{self.base_url}/{key}"
|
||||
|
||||
def get_download_url(self, key: str, expires: int = 3600) -> str:
|
||||
try:
|
||||
return self.client.get_presigned_download_url(
|
||||
Bucket=self.bucket,
|
||||
Key=key,
|
||||
Expired=expires,
|
||||
)
|
||||
except (CosClientError, CosServiceError) as exc:
|
||||
retryable = _is_retryable_cos_error(exc)
|
||||
request_id = (
|
||||
exc.get_request_id() if isinstance(exc, CosServiceError) else ""
|
||||
)
|
||||
raise CosDownloadUrlError(
|
||||
f"COS presigned URL failed: key={key}, error={exc}",
|
||||
retryable=retryable,
|
||||
request_id=request_id,
|
||||
) from exc
|
||||
|
||||
@classmethod
|
||||
def from_settings(cls, settings) -> "TencentCosStorageService":
|
||||
config = (
|
||||
getattr(settings, "tencent_cos_secret_id", "") or "",
|
||||
getattr(settings, "tencent_cos_secret_key", "") or "",
|
||||
getattr(settings, "tencent_cos_region", "") or "",
|
||||
getattr(settings, "tencent_cos_bucket", "") or "",
|
||||
getattr(settings, "tencent_cos_base_url", "") or "",
|
||||
getattr(settings, "tencent_cos_token", "") or "",
|
||||
)
|
||||
if cls._instance is None or cls._instance_config != config:
|
||||
cls._instance = cls(
|
||||
secret_id=config[0],
|
||||
secret_key=config[1],
|
||||
region=config[2],
|
||||
bucket=config[3],
|
||||
base_url=config[4],
|
||||
token=config[5],
|
||||
)
|
||||
cls._instance_config = config
|
||||
return cls._instance
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> "TencentCosStorageService":
|
||||
from app.core.config import settings
|
||||
|
||||
return cls.from_settings(settings)
|
||||
Reference in New Issue
Block a user