Compare commits
No commits in common. "112a41d3720fd32df0c7c1c633b37486c9805964" and "fafb0aee4f32e0d88b7744dc8f8bebfb114e1e45" have entirely different histories.
112a41d372
...
fafb0aee4f
@ -66,8 +66,6 @@ def setup_logging(quiet_mode=False):
|
||||
class DouyinAutoScheduler:
|
||||
def __init__(self):
|
||||
self.is_running = False
|
||||
# 创建logger实例
|
||||
self.logger = logging.getLogger(__name__)
|
||||
|
||||
def _normalize_play_vv(self, play_vv):
|
||||
"""标准化播放量数据类型，将字符串转换为数字"""
|
||||
@ -80,84 +78,29 @@ class DouyinAutoScheduler:
|
||||
return 0
|
||||
return play_vv
|
||||
|
||||
def check_browser_login_status(self):
|
||||
"""检查浏览器登录状态，如果没有登录则提示用户登录"""
|
||||
try:
|
||||
import os
|
||||
script_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
profile_dir = os.path.join(script_dir, 'config', 'chrome_profile_timer', 'douyin_persistent')
|
||||
|
||||
|
||||
# 检查配置文件是否为空（可能未登录）
|
||||
import glob
|
||||
profile_files = glob.glob(os.path.join(profile_dir, "*"))
|
||||
if len(profile_files) < 5: # 如果文件太少，可能未登录
|
||||
print("⚠️ 检测到定时器浏览器可能未登录")
|
||||
print(" 请在浏览器中完成抖音登录，并导航到【我的】→【收藏】→【合集】页面")
|
||||
print(" 完成后按回车键继续...")
|
||||
input()
|
||||
else:
|
||||
print("✅ 定时器浏览器已配置，继续执行...")
|
||||
|
||||
except Exception as e:
|
||||
logging.warning(f"检查浏览器登录状态时出错: {e}")
|
||||
print("⚠️ 检查浏览器状态失败，请确保浏览器已正确配置")
|
||||
print(" 完成后按回车键继续...")
|
||||
input()
|
||||
|
||||
def _cleanup_chrome_processes(self):
|
||||
"""清理可能占用配置文件的Chrome进程"""
|
||||
try:
|
||||
import psutil
|
||||
import os
|
||||
|
||||
# 获取当前配置文件路径
|
||||
script_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
profile_dir = os.path.join(script_dir, 'config', 'chrome_profile_timer', 'douyin_persistent')
|
||||
|
||||
# 查找使用该配置文件的Chrome进程
|
||||
killed_processes = []
|
||||
for proc in psutil.process_iter(['pid', 'name', 'cmdline']):
|
||||
try:
|
||||
if proc.info['name'] and 'chrome' in proc.info['name'].lower():
|
||||
cmdline = proc.info['cmdline']
|
||||
if cmdline and any(profile_dir in arg for arg in cmdline):
|
||||
proc.terminate()
|
||||
killed_processes.append(proc.info['pid'])
|
||||
logging.info(f'终止占用配置文件的Chrome进程: PID {proc.info["pid"]}')
|
||||
except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
|
||||
continue
|
||||
|
||||
# 等待进程终止
|
||||
if killed_processes:
|
||||
import time
|
||||
time.sleep(2)
|
||||
|
||||
return len(killed_processes) > 0
|
||||
|
||||
except ImportError:
|
||||
# 如果没有psutil，使用系统命令
|
||||
try:
|
||||
import subprocess
|
||||
import os
|
||||
|
||||
script_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
profile_dir = os.path.join(script_dir, 'config', 'chrome_profile_timer', 'douyin_persistent')
|
||||
|
||||
# 使用taskkill命令终止Chrome进程
|
||||
result = subprocess.run(['taskkill', '/F', '/IM', 'chrome.exe'], capture_output=True, text=True)
|
||||
if result.returncode == 0:
|
||||
logging.info('使用系统命令终止Chrome进程')
|
||||
return True
|
||||
else:
|
||||
logging.warning('无法终止Chrome进程')
|
||||
return False
|
||||
except Exception as e:
|
||||
logging.warning(f'系统命令清理Chrome进程失败: {e}')
|
||||
return False
|
||||
except Exception as e:
|
||||
logging.warning(f'清理Chrome进程时出错: {e}')
|
||||
return False
|
||||
def _deduplicate_videos_by_mix_name(self, videos, include_rank=False):
|
||||
"""按短剧名称去重，保留播放量最高的记录"""
|
||||
unique_data = {}
|
||||
for video in videos:
|
||||
mix_name = video.get("mix_name", "")
|
||||
if mix_name:
|
||||
# 标准化播放量数据类型
|
||||
play_vv = self._normalize_play_vv(video.get("play_vv", 0))
|
||||
|
||||
if mix_name not in unique_data or play_vv > unique_data[mix_name].get("play_vv", 0):
|
||||
if include_rank:
|
||||
# 用于昨天数据的格式
|
||||
unique_data[mix_name] = {
|
||||
"play_vv": play_vv,
|
||||
"video_id": str(video.get("_id", "")),
|
||||
"rank": 0 # 稍后计算排名
|
||||
}
|
||||
else:
|
||||
# 用于今天数据的格式，直接更新原视频对象
|
||||
video["play_vv"] = play_vv
|
||||
unique_data[mix_name] = video
|
||||
|
||||
return unique_data
|
||||
|
||||
def run_douyin_scraper(self):
|
||||
"""执行抖音播放量抓取任务"""
|
||||
@ -171,14 +114,14 @@ class DouyinAutoScheduler:
|
||||
scraper = DouyinPlayVVScraper(
|
||||
start_url="https://www.douyin.com/user/self?showTab=favorite_collection&showSubTab=compilation",
|
||||
auto_continue=True,
|
||||
duration_s=60 # 增加到60秒，给更多时间收集数据
|
||||
duration_s=60
|
||||
)
|
||||
|
||||
print("开始执行抓取任务...")
|
||||
|
||||
print("📁 开始执行抓取任务...")
|
||||
logging.info("📁 开始执行抓取任务...")
|
||||
scraper.run()
|
||||
|
||||
print("抖音播放量抓取任务执行成功")
|
||||
print("✅ 抖音播放量抓取任务执行成功")
|
||||
logging.info("✅ 抖音播放量抓取任务执行成功")
|
||||
|
||||
# 数据抓取完成后，自动生成当日榜单
|
||||
@ -237,25 +180,13 @@ class DouyinAutoScheduler:
|
||||
today_videos_raw = list(douyin_collection.find({"batch_time": latest_batch_time}).sort("play_vv", -1))
|
||||
logging.info(f"📊 最新批次数据数量: {len(today_videos_raw)}")
|
||||
|
||||
# 按短剧ID去重，每个短剧只保留播放量最高的一条
|
||||
# 🚫 过滤掉空的或无效的mix_id和播放量为0的记录
|
||||
# 按短剧名称去重，每个短剧只保留播放量最高的一条
|
||||
unique_videos = {}
|
||||
for video in today_videos_raw:
|
||||
mix_id = video.get("mix_id", "").strip()
|
||||
mix_name = video.get("mix_name", "").strip()
|
||||
play_vv = video.get("play_vv", 0)
|
||||
|
||||
# 过滤掉空的或无效的mix_id
|
||||
if not mix_id or mix_id == "" or mix_id.lower() == "null":
|
||||
continue
|
||||
|
||||
# 过滤掉播放量为0或无效的记录
|
||||
if play_vv <= 0:
|
||||
continue
|
||||
|
||||
if mix_id not in unique_videos or play_vv > unique_videos[mix_id].get("play_vv", 0):
|
||||
unique_videos[mix_id] = video
|
||||
|
||||
mix_name = video.get("mix_name", "")
|
||||
if mix_name and (mix_name not in unique_videos or video.get("play_vv", 0) > unique_videos[mix_name].get("play_vv", 0)):
|
||||
unique_videos[mix_name] = video
|
||||
|
||||
today_videos = list(unique_videos.values())
|
||||
|
||||
logging.info(f"📊 今日数据去重后：{len(today_videos)} 个独特短剧（原始数据：{len(today_videos_raw)} 条）")
|
||||
@ -282,28 +213,16 @@ class DouyinAutoScheduler:
|
||||
"batch_time": yesterday_batch_time
|
||||
}).sort("play_vv", -1))
|
||||
|
||||
# 按短剧ID去重，每个短剧只保留播放量最高的一条
|
||||
# 🚫 过滤掉空的或无效的mix_id和播放量为0的记录
|
||||
# 按短剧名称去重，每个短剧只保留播放量最高的一条
|
||||
unique_yesterday_videos = {}
|
||||
for video in yesterday_videos_raw:
|
||||
mix_id = video.get("mix_id", "").strip()
|
||||
mix_name = video.get("mix_name", "").strip()
|
||||
play_vv = video.get("play_vv", 0)
|
||||
|
||||
# 过滤掉空的或无效的mix_id
|
||||
if not mix_id or mix_id == "" or mix_id.lower() == "null":
|
||||
continue
|
||||
|
||||
# 过滤掉播放量为0或无效的记录
|
||||
if play_vv <= 0:
|
||||
continue
|
||||
|
||||
if mix_id not in unique_yesterday_videos or play_vv > unique_yesterday_videos[mix_id].get("play_vv", 0):
|
||||
unique_yesterday_videos[mix_id] = video
|
||||
|
||||
# 将昨天的数据转换为字典，以短剧ID为键
|
||||
for mix_id, video in unique_yesterday_videos.items():
|
||||
yesterday_data[mix_id] = {
|
||||
mix_name = video.get("mix_name", "")
|
||||
if mix_name and (mix_name not in unique_yesterday_videos or video.get("play_vv", 0) > unique_yesterday_videos[mix_name].get("play_vv", 0)):
|
||||
unique_yesterday_videos[mix_name] = video
|
||||
|
||||
# 将昨天的数据转换为字典，以短剧名称为键
|
||||
for mix_name, video in unique_yesterday_videos.items():
|
||||
yesterday_data[mix_name] = {
|
||||
"rank": 0, # 原始数据没有排名，设为0
|
||||
"play_vv": video.get("play_vv", 0),
|
||||
"video_id": str(video.get("_id", ""))
|
||||
@ -325,23 +244,23 @@ class DouyinAutoScheduler:
|
||||
play_vv_change_rate = 0
|
||||
is_new = True
|
||||
|
||||
mix_id = video.get("mix_id", "")
|
||||
if mix_id in yesterday_data:
|
||||
mix_name = video.get("mix_name", "")
|
||||
if mix_name in yesterday_data:
|
||||
is_new = False
|
||||
yesterday_play_vv = yesterday_data[mix_id]["play_vv"]
|
||||
|
||||
yesterday_play_vv = yesterday_data[mix_name]["play_vv"]
|
||||
|
||||
# 计算播放量变化
|
||||
play_vv_change = current_play_vv - yesterday_play_vv
|
||||
if yesterday_play_vv > 0:
|
||||
play_vv_change_rate = round((play_vv_change / yesterday_play_vv) * 100, 2)
|
||||
|
||||
|
||||
# 创建包含增长数据的视频项
|
||||
video_with_growth = {
|
||||
"video": video,
|
||||
"play_vv_change": play_vv_change,
|
||||
"play_vv_change_rate": play_vv_change_rate,
|
||||
"is_new": is_new,
|
||||
"yesterday_data": yesterday_data.get(mix_id, {})
|
||||
"yesterday_data": yesterday_data.get(mix_name, {})
|
||||
}
|
||||
videos_with_growth.append(video_with_growth)
|
||||
|
||||
@ -358,125 +277,29 @@ class DouyinAutoScheduler:
|
||||
"data": []
|
||||
}
|
||||
|
||||
# 获取Rankings_management集合用于补充详细信息
|
||||
rankings_management_collection = db['Rankings_management']
|
||||
|
||||
# 生成排序后的榜单数据
|
||||
rank = 1 # 使用独立的排名计数器
|
||||
for item in videos_with_growth:
|
||||
for i, item in enumerate(videos_with_growth, 1):
|
||||
video = item["video"]
|
||||
video_id = str(video.get("_id", ""))
|
||||
current_play_vv = video.get("play_vv", 0)
|
||||
mix_name = video.get("mix_name", "").strip()
|
||||
|
||||
# 🚫 跳过无效数据：确保mix_name不为空且播放量大于0
|
||||
# 注意：这些数据应该已经在去重阶段被过滤掉了，这里是双重保险
|
||||
if not mix_name or mix_name == "" or mix_name.lower() == "null":
|
||||
self.logger.warning(f"跳过空的mix_name记录，video_id: {video_id}")
|
||||
continue
|
||||
|
||||
if current_play_vv <= 0:
|
||||
self.logger.warning(f"跳过播放量无效的记录: mix_name={mix_name}, play_vv={current_play_vv}")
|
||||
continue
|
||||
mix_name = video.get("mix_name", "")
|
||||
|
||||
# 计算排名变化（基于昨天的排名）
|
||||
rank_change = 0
|
||||
if not item["is_new"] and item["yesterday_data"]:
|
||||
yesterday_rank = item["yesterday_data"].get("rank", 0)
|
||||
rank_change = yesterday_rank - rank # 使用当前排名计数器
|
||||
rank_change = yesterday_rank - i
|
||||
|
||||
# 🔍 从Rankings_management获取详细信息（按日期和mix_name查询）
|
||||
today_str = datetime.now().strftime('%Y-%m-%d')
|
||||
management_data = rankings_management_collection.find_one({
|
||||
"mix_name": mix_name,
|
||||
"$or": [
|
||||
{"created_at": {"$gte": datetime.strptime(today_str, '%Y-%m-%d'),
|
||||
"$lt": datetime.strptime(today_str, '%Y-%m-%d') + timedelta(days=1)}},
|
||||
{"last_updated": {"$gte": datetime.strptime(today_str, '%Y-%m-%d'),
|
||||
"$lt": datetime.strptime(today_str, '%Y-%m-%d') + timedelta(days=1)}}
|
||||
]
|
||||
})
|
||||
|
||||
# 🔑 如果今天没有数据，查询昨天的 Rankings_management（仅用于获取分类字段和锁定状态）
|
||||
classification_data = None
|
||||
if not management_data:
|
||||
# 查询昨天的 Rankings_management
|
||||
yesterday_start = datetime.strptime(yesterday_str, '%Y-%m-%d')
|
||||
yesterday_end = yesterday_start + timedelta(days=1)
|
||||
classification_data = rankings_management_collection.find_one({
|
||||
"mix_name": mix_name,
|
||||
"$or": [
|
||||
{"created_at": {"$gte": yesterday_start, "$lt": yesterday_end}},
|
||||
{"last_updated": {"$gte": yesterday_start, "$lt": yesterday_end}}
|
||||
]
|
||||
})
|
||||
if classification_data:
|
||||
novel_ids = classification_data.get('Novel_IDs', [])
|
||||
anime_ids = classification_data.get('Anime_IDs', [])
|
||||
drama_ids = classification_data.get('Drama_IDs', [])
|
||||
logging.info(f"📋 今天没有数据，从昨天的 Rankings_management 获取分类: {mix_name}")
|
||||
logging.info(f" - Novel_IDs: {novel_ids}")
|
||||
logging.info(f" - Anime_IDs: {anime_ids}")
|
||||
logging.info(f" - Drama_IDs: {drama_ids}")
|
||||
logging.info(f" - last_updated: {classification_data.get('last_updated')}")
|
||||
else:
|
||||
logging.warning(f"⚠️ 今天和昨天都没有数据: {mix_name}")
|
||||
|
||||
ranking_item = {
|
||||
# 🎯 核心榜单字段
|
||||
"rank": rank, # 使用排名计数器
|
||||
"rank": i,
|
||||
"title": mix_name,
|
||||
"mix_name": mix_name,
|
||||
"play_vv": current_play_vv,
|
||||
"series_author": video.get("series_author", ""),
|
||||
"author": video.get("author", ""),
|
||||
"video_id": video_id,
|
||||
"video_url": video.get("video_url", ""),
|
||||
"cover_image_url": video.get("cover_image_url", ""),
|
||||
"playcount_str": video.get("playcount", ""),
|
||||
|
||||
# 📋 从Rankings_management获取的详细字段
|
||||
"batch_id": management_data.get("batch_id", "") if management_data else "",
|
||||
"batch_time": management_data.get("batch_time") if management_data else None,
|
||||
"item_sequence": management_data.get("item_sequence", 0) if management_data else 0,
|
||||
"mix_id": video.get("mix_id", ""), # 直接从原始数据获取mix_id
|
||||
"playcount": management_data.get("playcount", "") if management_data else "",
|
||||
"request_id": management_data.get("request_id", "") if management_data else "",
|
||||
"cover_image_url_original": management_data.get("cover_image_url_original", "") if management_data else "",
|
||||
"cover_upload_success": management_data.get("cover_upload_success", True) if management_data else True,
|
||||
"cover_backup_urls": management_data.get("cover_backup_urls", []) if management_data else [],
|
||||
"desc": management_data.get("desc", "") if management_data else "",
|
||||
"updated_to_episode": management_data.get("updated_to_episode", 0) if management_data else 0,
|
||||
"episode_video_ids": management_data.get("episode_video_ids", []) if management_data else [],
|
||||
"episode_details": management_data.get("episode_details", []) if management_data else [],
|
||||
"data_status": management_data.get("data_status", "") if management_data else "",
|
||||
"realtime_saved": management_data.get("realtime_saved", True) if management_data else True,
|
||||
"created_at": management_data.get("created_at") if management_data else None,
|
||||
"last_updated": management_data.get("last_updated") if management_data else None,
|
||||
# 🔑 分类字段：区分今天数据和历史数据
|
||||
# - 如果今天有数据：从今天的数据获取所有字段
|
||||
# - 如果今天没有数据：只从历史记录获取分类字段和锁定状态，其他字段为空
|
||||
"Manufacturing_Field": management_data.get("Manufacturing_Field", "") if management_data else "",
|
||||
"Copyright_field": management_data.get("Copyright_field", "") if management_data else "",
|
||||
"Novel_IDs": (
|
||||
management_data.get("Novel_IDs", []) if management_data
|
||||
else (classification_data.get("Novel_IDs", []) if classification_data else [])
|
||||
),
|
||||
"Anime_IDs": (
|
||||
management_data.get("Anime_IDs", []) if management_data
|
||||
else (classification_data.get("Anime_IDs", []) if classification_data else [])
|
||||
),
|
||||
"Drama_IDs": (
|
||||
management_data.get("Drama_IDs", []) if management_data
|
||||
else (classification_data.get("Drama_IDs", []) if classification_data else [])
|
||||
),
|
||||
|
||||
# 🔒 锁定状态：同样区分今天数据和历史数据
|
||||
"field_lock_status": (
|
||||
management_data.get("field_lock_status", {}) if management_data
|
||||
else (classification_data.get("field_lock_status", {}) if classification_data else {})
|
||||
),
|
||||
|
||||
# 📊 时间轴对比数据（重要：包含播放量差值）
|
||||
# 时间轴对比数据
|
||||
"timeline_data": {
|
||||
"is_new": item["is_new"],
|
||||
"rank_change": rank_change,
|
||||
@ -488,7 +311,6 @@ class DouyinAutoScheduler:
|
||||
}
|
||||
|
||||
comprehensive_ranking["data"].append(ranking_item)
|
||||
rank += 1 # 递增排名计数器
|
||||
|
||||
# 为每次计算添加唯一的时间戳，确保数据唯一性
|
||||
current_timestamp = datetime.now()
|
||||
@ -508,29 +330,6 @@ class DouyinAutoScheduler:
|
||||
logging.info(f"📝 创建了新的今日榜单数据（第{existing_count + 1}次计算，包含最新差值）")
|
||||
logging.info(f"🔖 计算ID: {comprehensive_ranking['calculation_id']}")
|
||||
|
||||
# 📊 检查数据完整性：统计从Rankings_management成功获取详细信息的项目数量
|
||||
total_items = len(comprehensive_ranking["data"])
|
||||
items_with_management_data = 0
|
||||
items_with_manufacturing = 0
|
||||
items_with_copyright = 0
|
||||
|
||||
for item in comprehensive_ranking["data"]:
|
||||
# 检查是否从Rankings_management获取到了数据
|
||||
if item.get("batch_id") or item.get("desc") or item.get("Manufacturing_Field") or item.get("Copyright_field"):
|
||||
items_with_management_data += 1
|
||||
if item.get("Manufacturing_Field"):
|
||||
items_with_manufacturing += 1
|
||||
if item.get("Copyright_field"):
|
||||
items_with_copyright += 1
|
||||
|
||||
print(f"数据完整性统计:")
|
||||
print(f" 总项目数: {total_items}")
|
||||
print(f" 从Rankings_management获取到详细信息: {items_with_management_data}")
|
||||
print(f" 包含Manufacturing_Field: {items_with_manufacturing}")
|
||||
print(f" 包含Copyright_field: {items_with_copyright}")
|
||||
|
||||
logging.info(f"📊 数据完整性: 总{total_items}项，获取详细信息{items_with_management_data}项，Manufacturing_Field: {items_with_manufacturing}，Copyright_field: {items_with_copyright}")
|
||||
|
||||
# 统计信息
|
||||
new_count = sum(1 for item in comprehensive_ranking["data"] if item["timeline_data"]["is_new"])
|
||||
print(f"✅ 时间轴对比榜单生成成功")
|
||||
@ -559,165 +358,13 @@ class DouyinAutoScheduler:
|
||||
import traceback
|
||||
logging.error(f"详细错误信息: {traceback.format_exc()}")
|
||||
|
||||
def check_and_sync_missing_fields(self):
|
||||
"""实时检查并同步当天缺失字段"""
|
||||
try:
|
||||
from database import db
|
||||
|
||||
# 只检查当天的数据
|
||||
today = date.today()
|
||||
today_str = today.strftime('%Y-%m-%d')
|
||||
|
||||
# 首先检查 Rankings_management 是否有当天的数据
|
||||
rankings_management_collection = db['Rankings_management']
|
||||
management_count = rankings_management_collection.count_documents({})
|
||||
|
||||
if management_count == 0:
|
||||
# Rankings_management 没有数据，说明还没有抓取，直接返回
|
||||
return
|
||||
|
||||
rankings_collection = db['Ranking_storage']
|
||||
key_fields = ['Manufacturing_Field', 'Copyright_field', 'desc', 'series_author']
|
||||
|
||||
# 检查今天是否有缺失字段的数据
|
||||
missing_conditions = []
|
||||
for field in key_fields:
|
||||
missing_conditions.extend([
|
||||
{field: {"$exists": False}},
|
||||
{field: None},
|
||||
{field: ""}
|
||||
])
|
||||
|
||||
today_missing_count = rankings_collection.count_documents({
|
||||
"date": today_str,
|
||||
"$or": missing_conditions
|
||||
})
|
||||
|
||||
# 如果今天没有缺失数据，静默返回
|
||||
if today_missing_count == 0:
|
||||
return
|
||||
|
||||
logging.info(f"🔍 检测到今天有 {today_missing_count} 条缺失字段，Rankings_management有 {management_count} 条数据，开始实时同步...")
|
||||
|
||||
# 只处理当天的数据
|
||||
dates_to_check = [today_str]
|
||||
|
||||
total_missing = 0
|
||||
total_synced = 0
|
||||
|
||||
for check_date in dates_to_check:
|
||||
# 查询该日期缺失字段的数据
|
||||
rankings_collection = db['Ranking_storage']
|
||||
|
||||
# 检查多个关键字段（包括新增的分类字段）
|
||||
key_fields = ['Manufacturing_Field', 'Copyright_field', 'desc', 'series_author', 'Novel_IDs', 'Anime_IDs', 'Drama_IDs']
|
||||
missing_conditions = []
|
||||
|
||||
for field in key_fields:
|
||||
missing_conditions.extend([
|
||||
{field: {"$exists": False}},
|
||||
{field: None},
|
||||
{field: ""}
|
||||
])
|
||||
|
||||
missing_query = {
|
||||
"date": check_date,
|
||||
"$or": missing_conditions
|
||||
}
|
||||
|
||||
missing_count = rankings_collection.count_documents(missing_query)
|
||||
|
||||
# 详细统计每个字段的缺失情况
|
||||
field_stats = {}
|
||||
total_items = rankings_collection.count_documents({"date": check_date})
|
||||
|
||||
for field in key_fields:
|
||||
missing_field_count = rankings_collection.count_documents({
|
||||
"date": check_date,
|
||||
"$or": [
|
||||
{field: {"$exists": False}},
|
||||
{field: None},
|
||||
{field: ""}
|
||||
]
|
||||
})
|
||||
field_stats[field] = {
|
||||
"missing": missing_field_count,
|
||||
"completion_rate": ((total_items - missing_field_count) / total_items * 100) if total_items > 0 else 0
|
||||
}
|
||||
|
||||
if missing_count > 0:
|
||||
logging.info(f"📅 今日({check_date}): 发现 {missing_count} 条记录缺失字段（总计 {total_items} 条）")
|
||||
|
||||
# 输出详细的字段统计
|
||||
for field, stats in field_stats.items():
|
||||
if stats["missing"] > 0:
|
||||
logging.info(f" - {field}: 缺失 {stats['missing']} 条 ({stats['completion_rate']:.1f}% 完整)")
|
||||
|
||||
total_missing += missing_count
|
||||
|
||||
# 尝试同步
|
||||
try:
|
||||
from routers.rank_api_routes import sync_ranking_storage_fields
|
||||
|
||||
# 使用改进的重试机制
|
||||
sync_result = sync_ranking_storage_fields(
|
||||
target_date=check_date,
|
||||
force_update=False,
|
||||
max_retries=2, # 定期检查时重试2次
|
||||
retry_delay=15 # 15秒重试间隔
|
||||
)
|
||||
|
||||
if sync_result.get("success", False):
|
||||
stats = sync_result.get("stats", {})
|
||||
synced = stats.get("updated_items", 0)
|
||||
retry_count = stats.get("retry_count", 0)
|
||||
pending_final = stats.get("pending_items_final", 0)
|
||||
|
||||
total_synced += synced
|
||||
if synced > 0:
|
||||
logging.info(f"✅ 今日({check_date}): 成功同步 {synced} 条记录")
|
||||
|
||||
if retry_count > 0:
|
||||
logging.info(f"🔄 今日({check_date}): 使用了 {retry_count} 次重试")
|
||||
|
||||
if pending_final > 0:
|
||||
logging.warning(f"⚠️ 今日({check_date}): {pending_final} 条记录在 Rankings_management 中仍未找到")
|
||||
else:
|
||||
logging.warning(f"⚠️ 今日({check_date}): 同步失败 - {sync_result.get('message', '')}")
|
||||
|
||||
except Exception as sync_error:
|
||||
logging.error(f"💥 今日({check_date}): 同步过程出错 - {sync_error}")
|
||||
else:
|
||||
if total_items > 0:
|
||||
logging.info(f"📅 {check_date}: 所有字段完整（总计 {total_items} 条记录）")
|
||||
# 显示完整性统计
|
||||
for field, stats in field_stats.items():
|
||||
logging.info(f" - {field}: {stats['completion_rate']:.1f}% 完整")
|
||||
else:
|
||||
logging.info(f"📅 {check_date}: 无数据")
|
||||
|
||||
if total_missing > 0:
|
||||
logging.info(f"🔍 当天同步完成：发现 {total_missing} 条缺失记录，成功同步 {total_synced} 条")
|
||||
print(f"🔍 当天字段同步：发现 {total_missing} 条缺失，同步 {total_synced} 条")
|
||||
else:
|
||||
# 当天没有缺失数据时，不输出日志（静默模式）
|
||||
pass
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f"💥 检查缺失字段时发生异常: {e}")
|
||||
import traceback
|
||||
logging.error(f"详细错误信息: {traceback.format_exc()}")
|
||||
|
||||
def setup_schedule(self):
|
||||
"""设置定时任务"""
|
||||
# 每小时的整点执行抖音播放量抓取
|
||||
schedule.every().hour.at(":00").do(self.run_douyin_scraper)
|
||||
|
||||
# 每1分钟检查一次缺失字段并尝试同步（实时同步）
|
||||
schedule.every(1).minutes.do(self.check_and_sync_missing_fields)
|
||||
|
||||
logging.info(f"⏰ 定时器已设置：每小时整点执行抖音播放量抓取")
|
||||
logging.info(f"⏰ 定时器已设置：每1分钟检查缺失字段并同步（实时模式）")
|
||||
|
||||
def show_next_run(self):
|
||||
"""显示下次执行时间"""
|
||||
|
||||
@ -19,13 +19,9 @@
|
||||
{
|
||||
"video_id": "7471924777410645283",
|
||||
"episode_num": 0
|
||||
},
|
||||
{
|
||||
"video_id": "7472791705268325641",
|
||||
"episode_num": 0
|
||||
}
|
||||
],
|
||||
"total_count": 6,
|
||||
"last_update": "2025-11-06T17:43:54.929209",
|
||||
"total_count": 5,
|
||||
"last_update": "2025-10-22T09:55:36.943794",
|
||||
"mix_name": "《青蛇传》"
|
||||
}
|
||||
@ -143,21 +143,9 @@
|
||||
{
|
||||
"video_id": "7558378239337467174",
|
||||
"episode_num": 0
|
||||
},
|
||||
{
|
||||
"video_id": "7567050545257516331",
|
||||
"episode_num": 0
|
||||
},
|
||||
{
|
||||
"video_id": "7568152326477942022",
|
||||
"episode_num": 0
|
||||
},
|
||||
{
|
||||
"video_id": "7569217928420183332",
|
||||
"episode_num": 0
|
||||
}
|
||||
],
|
||||
"total_count": 39,
|
||||
"last_update": "2025-11-06T11:06:44.598400",
|
||||
"total_count": 36,
|
||||
"last_update": "2025-10-22T09:55:32.073567",
|
||||
"mix_name": "末世系列"
|
||||
}
|
||||
@ -47,17 +47,9 @@
|
||||
{
|
||||
"video_id": "7548447317729234239",
|
||||
"episode_num": 0
|
||||
},
|
||||
{
|
||||
"video_id": "7568747381357808923",
|
||||
"episode_num": 0
|
||||
},
|
||||
{
|
||||
"video_id": "7568800392985791784",
|
||||
"episode_num": 0
|
||||
}
|
||||
],
|
||||
"total_count": 14,
|
||||
"last_update": "2025-11-06T17:48:06.014161",
|
||||
"total_count": 12,
|
||||
"last_update": "2025-10-22T09:55:50.726907",
|
||||
"mix_name": "青云修仙传"
|
||||
}
|
||||
@ -107,17 +107,9 @@
|
||||
{
|
||||
"video_id": "7560551213957500195",
|
||||
"episode_num": 0
|
||||
},
|
||||
{
|
||||
"video_id": "7562056353343966464",
|
||||
"episode_num": 0
|
||||
},
|
||||
{
|
||||
"video_id": "7567981488823318927",
|
||||
"episode_num": 0
|
||||
}
|
||||
],
|
||||
"total_count": 29,
|
||||
"last_update": "2025-11-06T17:15:32.747557",
|
||||
"total_count": 27,
|
||||
"last_update": "2025-10-22T09:56:16.947762",
|
||||
"mix_name": "绝境逆袭"
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
BIN
frontend/public/favicon.ico
Normal file
BIN
frontend/public/favicon.ico
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 4.2 KiB |
5
frontend/public/placeholder-poster.svg
Normal file
5
frontend/public/placeholder-poster.svg
Normal file
@ -0,0 +1,5 @@
|
||||
<svg width="60" height="80" xmlns="http://www.w3.org/2000/svg">
|
||||
<rect width="60" height="80" fill="#f0f0f0" stroke="#ddd" stroke-width="1"/>
|
||||
<text x="30" y="35" text-anchor="middle" font-family="Arial" font-size="8" fill="#999">暂无</text>
|
||||
<text x="30" y="50" text-anchor="middle" font-family="Arial" font-size="8" fill="#999">图片</text>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 355 B |
File diff suppressed because it is too large
Load Diff
@ -1,11 +1,7 @@
|
||||
<script setup>
|
||||
import { ref, onMounted, computed } from 'vue'
|
||||
import { useRouter, useRoute } from 'vue-router'
|
||||
import axios from 'axios'
|
||||
|
||||
const router = useRouter()
|
||||
const route = useRoute()
|
||||
|
||||
// 响应式数据
|
||||
const currentTab = ref('ranking') // 'ranking' 或 'news'
|
||||
const rankingData = ref([])
|
||||
@ -63,22 +59,14 @@ const generateDateOptions = () => {
|
||||
const fetchRankingData = async () => {
|
||||
loading.value = true
|
||||
try {
|
||||
// 构建API参数
|
||||
const params = {
|
||||
page: currentPage.value,
|
||||
limit: 100,
|
||||
sort: 'growth',
|
||||
start_date: selectedDate.value,
|
||||
end_date: selectedDate.value
|
||||
}
|
||||
|
||||
// 如果选择了特定分类，添加分类参数
|
||||
if (selectedCategory.value !== 'all') {
|
||||
params.classification_type = selectedCategory.value
|
||||
}
|
||||
|
||||
const response = await axios.get('http://localhost:5001/api/rank/videos', {
|
||||
params: params
|
||||
params: {
|
||||
page: currentPage.value,
|
||||
limit: 20,
|
||||
sort: 'growth',
|
||||
start_date: selectedDate.value,
|
||||
end_date: selectedDate.value
|
||||
}
|
||||
})
|
||||
|
||||
if (response.data.success) {
|
||||
@ -86,7 +74,6 @@ const fetchRankingData = async () => {
|
||||
totalPages.value = response.data.pagination.pages
|
||||
// 获取后端返回的更新时间
|
||||
updateTime.value = response.data.update_time || ''
|
||||
console.log(`获取${selectedCategory.value === 'all' ? '全部' : selectedCategory.value}分类数据成功，共${response.data.data.length}条`)
|
||||
} else {
|
||||
console.error('获取数据失败:', response.data.message)
|
||||
rankingData.value = []
|
||||
@ -225,9 +212,7 @@ const selectDate = (dateValue) => {
|
||||
// 切换分类
|
||||
const switchCategory = (category) => {
|
||||
selectedCategory.value = category
|
||||
currentPage.value = 1 // 重置页码
|
||||
fetchRankingData() // 重新获取数据
|
||||
console.log(`切换到分类: ${category}`)
|
||||
// 这里可以添加分类筛选逻辑
|
||||
}
|
||||
|
||||
// 格式化日期显示（用于日榜标题）
|
||||
@ -252,11 +237,6 @@ const getRankBadgeClass = (rank) => {
|
||||
return 'rank-normal'
|
||||
}
|
||||
|
||||
// 跳转到后台管理
|
||||
// const goToAdmin = () => {
|
||||
// router.push('/admin')
|
||||
// }
|
||||
|
||||
// 页面加载时初始化
|
||||
onMounted(() => {
|
||||
initDate()
|
||||
@ -266,11 +246,8 @@ onMounted(() => {
|
||||
|
||||
<template>
|
||||
<div class="app">
|
||||
<!-- 路由视图 -->
|
||||
<router-view v-if="route.path !== '/'" />
|
||||
|
||||
<!-- 主容器 - 仅在首页显示 -->
|
||||
<div v-if="route.path === '/'" class="main-container">
|
||||
<!-- 主容器 -->
|
||||
<div class="main-container">
|
||||
<!-- 顶部标题区域 -->
|
||||
<div class="header-section">
|
||||
<div class="title-wrapper">
|
||||
@ -370,9 +347,9 @@ onMounted(() => {
|
||||
<img src="./images/版权icon.svg" alt="版权" class="detail-icon" />
|
||||
</div>
|
||||
<div class="detail-text">
|
||||
<p>剧场名：{{ item.series_author || '爱微剧场' }}</p>
|
||||
<p>承制：{{ item.Manufacturing_Field || '妙想制片厂' }}</p>
|
||||
<p>版权：{{ item.Copyright_field || '可梦' }}</p>
|
||||
<p>剧场名：爱微剧场</p>
|
||||
<p>承制：妙想制片厂</p>
|
||||
<p>版权：可梦</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@ -384,7 +361,7 @@ onMounted(() => {
|
||||
</div>
|
||||
<div class="stat-item">
|
||||
<img src="./images/点赞icon.svg" alt="点赞" class="stat-icon" />
|
||||
<span class="stat-value">{{ item.total_likes_formatted || '0' }}</span>
|
||||
<span class="stat-value">374W</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@ -425,8 +402,6 @@ onMounted(() => {
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
</div>
|
||||
</template>
|
||||
|
||||
@ -458,9 +433,11 @@ onMounted(() => {
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
gap: 10px;
|
||||
position: relative;
|
||||
}
|
||||
|
||||
.logo-icon {
|
||||
width: 40px;
|
||||
height: 40px;
|
||||
}
|
||||
.main-title {
|
||||
font-size: 24px;
|
||||
font-weight: bold;
|
||||
|
||||
@ -1,123 +0,0 @@
|
||||
<script setup>
|
||||
import { useRouter, useRoute } from 'vue-router'
|
||||
|
||||
const router = useRouter()
|
||||
const route = useRoute()
|
||||
|
||||
const navigateToHome = () => {
|
||||
router.push('/')
|
||||
}
|
||||
|
||||
const navigateToAdmin = () => {
|
||||
router.push('/admin')
|
||||
}
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div class="root-app">
|
||||
<!-- 导航栏 -->
|
||||
<nav class="navigation">
|
||||
<div class="nav-container">
|
||||
<div class="nav-brand">
|
||||
<h1>AI棒榜系统</h1>
|
||||
</div>
|
||||
<div class="nav-links">
|
||||
<button
|
||||
class="nav-btn"
|
||||
:class="{ active: route.path === '/admin' }"
|
||||
@click="navigateToAdmin"
|
||||
>
|
||||
后台管理
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</nav>
|
||||
|
||||
<!-- 路由视图 -->
|
||||
<main class="main-content">
|
||||
<router-view />
|
||||
</main>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<style scoped>
|
||||
.root-app {
|
||||
min-height: 100vh;
|
||||
background: #ebedf2;
|
||||
}
|
||||
|
||||
/* 导航栏样式 */
|
||||
.navigation {
|
||||
background: white;
|
||||
border-bottom: 1px solid #e0e0e0;
|
||||
position: sticky;
|
||||
top: 0;
|
||||
z-index: 100;
|
||||
}
|
||||
|
||||
.nav-container {
|
||||
max-width: 375px;
|
||||
margin: 0 auto;
|
||||
padding: 12px 16px;
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.nav-brand h1 {
|
||||
font-size: 18px;
|
||||
font-weight: bold;
|
||||
color: #333;
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
.nav-links {
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.nav-btn {
|
||||
padding: 8px 16px;
|
||||
border: 1px solid #ddd;
|
||||
background: white;
|
||||
color: #666;
|
||||
border-radius: 6px;
|
||||
cursor: pointer;
|
||||
font-size: 14px;
|
||||
font-weight: 500;
|
||||
transition: all 0.3s ease;
|
||||
}
|
||||
|
||||
.nav-btn:hover {
|
||||
background: #f8f9fa;
|
||||
border-color: #4a90e2;
|
||||
}
|
||||
|
||||
.nav-btn.active {
|
||||
background: #4a90e2;
|
||||
color: white;
|
||||
border-color: #4a90e2;
|
||||
}
|
||||
|
||||
/* 主内容区域 */
|
||||
.main-content {
|
||||
min-height: calc(100vh - 60px);
|
||||
}
|
||||
|
||||
/* 响应式设计 */
|
||||
@media (max-width: 480px) {
|
||||
.nav-container {
|
||||
max-width: 100%;
|
||||
padding: 10px 12px;
|
||||
}
|
||||
|
||||
.nav-brand h1 {
|
||||
font-size: 16px;
|
||||
}
|
||||
|
||||
.nav-btn {
|
||||
padding: 6px 12px;
|
||||
font-size: 13px;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
@ -1,8 +1,4 @@
|
||||
import { createApp } from 'vue'
|
||||
import './style.css'
|
||||
import App from './App.vue'
|
||||
import router from './router'
|
||||
|
||||
const app = createApp(App)
|
||||
|
||||
app.use(router).mount('#app')
|
||||
createApp(App).mount('#app')
|
||||
|
||||
@ -1,17 +0,0 @@
|
||||
import { createRouter, createWebHistory } from 'vue-router'
|
||||
import AdminPanel from '../AdminPanel.vue'
|
||||
|
||||
const routes = [
|
||||
{
|
||||
path: '/admin',
|
||||
name: 'Admin',
|
||||
component: AdminPanel
|
||||
}
|
||||
]
|
||||
|
||||
const router = createRouter({
|
||||
history: createWebHistory(),
|
||||
routes
|
||||
})
|
||||
|
||||
export default router
|
||||
@ -1,60 +0,0 @@
|
||||
/* 全局样式重置 */
|
||||
* {
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
body {
|
||||
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen',
|
||||
'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue',
|
||||
sans-serif;
|
||||
-webkit-font-smoothing: antialiased;
|
||||
-moz-osx-font-smoothing: grayscale;
|
||||
background-color: #ebedf2;
|
||||
color: #333;
|
||||
}
|
||||
|
||||
#app {
|
||||
width: 100%;
|
||||
min-height: 100vh;
|
||||
}
|
||||
|
||||
/* 通用按钮样式 */
|
||||
button {
|
||||
cursor: pointer;
|
||||
border: none;
|
||||
outline: none;
|
||||
font-family: inherit;
|
||||
}
|
||||
|
||||
/* 通用输入框样式 */
|
||||
input, textarea, select {
|
||||
font-family: inherit;
|
||||
outline: none;
|
||||
}
|
||||
|
||||
/* 滚动条样式 */
|
||||
::-webkit-scrollbar {
|
||||
width: 6px;
|
||||
}
|
||||
|
||||
::-webkit-scrollbar-track {
|
||||
background: #f1f1f1;
|
||||
}
|
||||
|
||||
::-webkit-scrollbar-thumb {
|
||||
background: #c1c1c1;
|
||||
border-radius: 3px;
|
||||
}
|
||||
|
||||
::-webkit-scrollbar-thumb:hover {
|
||||
background: #a8a8a8;
|
||||
}
|
||||
|
||||
/* 响应式设计 */
|
||||
@media (max-width: 480px) {
|
||||
body {
|
||||
font-size: 14px;
|
||||
}
|
||||
}
|
||||
@ -13,14 +13,4 @@ export default defineConfig({
|
||||
'@': fileURLToPath(new URL('./src', import.meta.url))
|
||||
},
|
||||
},
|
||||
server: {
|
||||
port: 5174,
|
||||
proxy: {
|
||||
'/api': {
|
||||
target: 'http://localhost:5001',
|
||||
changeOrigin: true,
|
||||
secure: false
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user