Merge commit '13b05ae252a12aa05bd63c7000020426c04b19f9' into xbh

backend/Timer_worker.py

@@ -249,9 +249,9 @@ class DouyinAutoScheduler:
                     if not mix_id or mix_id == "" or mix_id.lower() == "null":
                         continue
 
-                    # 过滤掉播放量为0或无效的记录
+                    # 注意：播放量为0的数据也会被保留，可能是新发布的短剧
                     if play_vv <= 0:
-                        continue
+                        logging.warning(f"⚠️ 发现播放量为0的数据: mix_name={mix_name}, play_vv={play_vv}，仍会保留")
 
                     if mix_id not in unique_videos or play_vv > unique_videos[mix_id].get("play_vv", 0):
                         unique_videos[mix_id] = video
@@ -283,7 +283,7 @@ class DouyinAutoScheduler:
                     }).sort("play_vv", -1))
                     
                     # 按短剧ID去重，每个短剧只保留播放量最高的一条
-                    # 🚫 过滤掉空的或无效的mix_id和播放量为0的记录
+                    # 🚫 过滤掉空的或无效的mix_id
                     unique_yesterday_videos = {}
                     for video in yesterday_videos_raw:
                         mix_id = video.get("mix_id", "").strip()
@@ -294,9 +294,9 @@ class DouyinAutoScheduler:
                         if not mix_id or mix_id == "" or mix_id.lower() == "null":
                             continue
 
-                        # 过滤掉播放量为0或无效的记录
+                        # 注意：播放量为0的数据也会被保留，可能是新发布的短剧
                         if play_vv <= 0:
-                            continue
+                            logging.warning(f"⚠️ 昨天数据中发现播放量为0: mix_name={mix_name}, play_vv={play_vv}，仍会保留")
 
                         if mix_id not in unique_yesterday_videos or play_vv > unique_yesterday_videos[mix_id].get("play_vv", 0):
                             unique_yesterday_videos[mix_id] = video
@@ -369,15 +369,14 @@ class DouyinAutoScheduler:
                         current_play_vv = video.get("play_vv", 0)
                         mix_name = video.get("mix_name", "").strip()
                         
-                        # 🚫 跳过无效数据：确保mix_name不为空且播放量大于0
+                        # 🚫 跳过无效数据：确保mix_name不为空
-                        # 注意：这些数据应该已经在去重阶段被过滤掉了，这里是双重保险
+                        # 注意：播放量为0的数据也会被保留，可能是新发布的短剧
                         if not mix_name or mix_name == "" or mix_name.lower() == "null":
                             self.logger.warning(f"跳过空的mix_name记录，video_id: {video_id}")
                             continue
                             
                         if current_play_vv <= 0:
-                            self.logger.warning(f"跳过播放量无效的记录: mix_name={mix_name}, play_vv={current_play_vv}")
+                            self.logger.warning(f"⚠️ 榜单中发现播放量为0的记录: mix_name={mix_name}, play_vv={current_play_vv}，仍会保留")
-                            continue
                         
                         # 计算排名变化（基于昨天的排名）
                         rank_change = 0

backend/handlers/Rankings/rank_data_scraper.py

@@ -809,91 +809,95 @@ class DouyinPlayVVScraper:
                         if isinstance(play_vv, (int, str)) and str(play_vv).isdigit():
                             vv = int(play_vv)
                             
-                            # 数据验证：确保合集名称不为空
+                            # 数据验证：确保有mix_id（按短剧ID去重，所以必须有mix_id）
-                            if not mix_name or mix_name.strip() == "":
+                            if not mix_id or mix_id.strip() == "":
-                                logging.warning(f"跳过缺少合集名称的数据: play_vv={vv}, mix_id={mix_id}")
+                                logging.warning(f"跳过缺少mix_id的数据: play_vv={vv}, mix_name={mix_name}")
-                                return
+                                # 跳过当前项，但继续递归解析其他数据（不使用return）
+                            else:
+                                # 如果mix_name为空，使用mix_id作为名称
+                                if not mix_name or mix_name.strip() == "":
+                                    mix_name = f"短剧_{mix_id}"
+                                    logging.warning(f"⚠️ mix_name为空，使用mix_id作为名称: {mix_name}")
+                                # 🔧 修复：不跳过播放量为0的数据，而是标记并保留
+                                # 这些数据可能是因为页面加载不完整，但合集本身是存在的
+                                # 警告信息移到去重检查之后，只有真正添加时才警告
                                 
-                            # 🔧 修复：不跳过播放量为0的数据，而是标记并保留
+                                # 构建合集链接
-                            # 这些数据可能是因为页面加载不完整，但合集本身是存在的
+                                video_url = f"https://www.douyin.com/collection/{mix_id}" if mix_id else ""
-                            # 警告信息移到去重检查之后，只有真正添加时才警告
                                 
-                            # 构建合集链接
+                                # 提取合集封面图片URL - 直接存储完整的图片链接
-                            video_url = f"https://www.douyin.com/collection/{mix_id}" if mix_id else ""
+                                cover_image_url = ""
+                                cover_image_backup_urls = []  # 备用链接列表
 
-                            # 提取合集封面图片URL - 直接存储完整的图片链接
+                                # 查找封面图片字段，优先获取完整的URL链接
-                            cover_image_url = ""
+                                if 'cover' in obj:
-                            cover_image_backup_urls = []  # 备用链接列表
+                                    cover = obj['cover']
+                                    if isinstance(cover, dict) and 'url_list' in cover and cover['url_list']:
+                                        # 主链接
+                                        cover_image_url = cover['url_list'][0]
+                                        # 备用链接
+                                        cover_image_backup_urls = cover['url_list'][1:] if len(cover['url_list']) > 1 else []
+                                    elif isinstance(cover, str):
+                                        cover_image_url = cover
+                                elif 'cover_url' in obj:
+                                    cover_url = obj['cover_url']
+                                    if isinstance(cover_url, dict) and 'url_list' in cover_url and cover_url['url_list']:
+                                        cover_image_url = cover_url['url_list'][0]
+                                        cover_image_backup_urls = cover_url['url_list'][1:] if len(cover_url['url_list']) > 1 else []
+                                    elif isinstance(cover_url, str):
+                                        cover_image_url = cover_url
+                                elif 'image' in obj:
+                                    image = obj['image']
+                                    if isinstance(image, dict) and 'url_list' in image and image['url_list']:
+                                        cover_image_url = image['url_list'][0]
+                                        cover_image_backup_urls = image['url_list'][1:] if len(image['url_list']) > 1 else []
+                                    elif isinstance(image, str):
+                                        cover_image_url = image
+                                elif 'pic' in obj:
+                                    pic = obj['pic']
+                                    if isinstance(pic, dict) and 'url_list' in pic and pic['url_list']:
+                                        cover_image_url = pic['url_list'][0]
+                                        cover_image_backup_urls = pic['url_list'][1:] if len(pic['url_list']) > 1 else []
+                                    elif isinstance(pic, str):
+                                        cover_image_url = pic
 
-                            # 查找封面图片字段，优先获取完整的URL链接
+                                # 提取新增的五个字段
-                            if 'cover' in obj:
+                                series_author = ""
-                                cover = obj['cover']
+                                desc = ""
-                                if isinstance(cover, dict) and 'url_list' in cover and cover['url_list']:
+                                updated_to_episode = 0
-                                    # 主链接
+                                manufacturing_field = ""  # 承制信息
-                                    cover_image_url = cover['url_list'][0]
+                                copyright_field = ""      # 版权信息
-                                    # 备用链接
-                                    cover_image_backup_urls = cover['url_list'][1:] if len(cover['url_list']) > 1 else []
-                                elif isinstance(cover, str):
-                                    cover_image_url = cover
-                            elif 'cover_url' in obj:
-                                cover_url = obj['cover_url']
-                                if isinstance(cover_url, dict) and 'url_list' in cover_url and cover_url['url_list']:
-                                    cover_image_url = cover_url['url_list'][0]
-                                    cover_image_backup_urls = cover_url['url_list'][1:] if len(cover_url['url_list']) > 1 else []
-                                elif isinstance(cover_url, str):
-                                    cover_image_url = cover_url
-                            elif 'image' in obj:
-                                image = obj['image']
-                                if isinstance(image, dict) and 'url_list' in image and image['url_list']:
-                                    cover_image_url = image['url_list'][0]
-                                    cover_image_backup_urls = image['url_list'][1:] if len(image['url_list']) > 1 else []
-                                elif isinstance(image, str):
-                                    cover_image_url = image
-                            elif 'pic' in obj:
-                                pic = obj['pic']
-                                if isinstance(pic, dict) and 'url_list' in pic and pic['url_list']:
-                                    cover_image_url = pic['url_list'][0]
-                                    cover_image_backup_urls = pic['url_list'][1:] if len(pic['url_list']) > 1 else []
-                                elif isinstance(pic, str):
-                                    cover_image_url = pic
 
-                            # 提取新增的五个字段
+                                # 提取合集作者/影视工作室
-                            series_author = ""
+                                if 'author' in obj:
-                            desc = ""
+                                    author = obj['author']
-                            updated_to_episode = 0
+                                    if isinstance(author, dict):
-                            manufacturing_field = ""  # 承制信息
+                                        # 尝试多个可能的作者字段
-                            copyright_field = ""      # 版权信息
+                                        series_author = (author.get('nickname') or 
+                                                       author.get('unique_id') or 
+                                                       author.get('short_id') or 
+                                                       author.get('name') or '')
+                                    elif isinstance(author, str):
+                                        series_author = author
+                                elif 'creator' in obj:
+                                    creator = obj['creator']
+                                    if isinstance(creator, dict):
+                                        series_author = (creator.get('nickname') or 
+                                                       creator.get('unique_id') or 
+                                                       creator.get('name') or '')
+                                    elif isinstance(creator, str):
+                                        series_author = creator
+                                elif 'user' in obj:
+                                    user = obj['user']
+                                    if isinstance(user, dict):
+                                        series_author = (user.get('nickname') or 
+                                                       user.get('unique_id') or 
+                                                       user.get('name') or '')
+                                    elif isinstance(user, str):
+                                        series_author = user
 
-                            # 提取合集作者/影视工作室
+                                # 提取合集描述 - 扩展更多可能的字段
-                            if 'author' in obj:
+                                description_fields = ['desc', 'share_info']  # 保持字段列表
-                                author = obj['author']
-                                if isinstance(author, dict):
-                                    # 尝试多个可能的作者字段
-                                    series_author = (author.get('nickname') or 
-                                                   author.get('unique_id') or 
-                                                   author.get('short_id') or 
-                                                   author.get('name') or '')
-                                elif isinstance(author, str):
-                                    series_author = author
-                            elif 'creator' in obj:
-                                creator = obj['creator']
-                                if isinstance(creator, dict):
-                                    series_author = (creator.get('nickname') or 
-                                                   creator.get('unique_id') or 
-                                                   creator.get('name') or '')
-                                elif isinstance(creator, str):
-                                    series_author = creator
-                            elif 'user' in obj:
-                                user = obj['user']
-                                if isinstance(user, dict):
-                                    series_author = (user.get('nickname') or 
-                                                   user.get('unique_id') or 
-                                                   user.get('name') or '')
-                                elif isinstance(user, str):
-                                    series_author = user
-
-                            # 提取合集描述 - 扩展更多可能的字段
-                            description_fields = ['desc', 'share_info']  # 保持字段列表
                             
                             # 先检查desc字段
                             if 'desc' in obj and obj['desc']:
@@ -999,9 +1003,9 @@ class DouyinPlayVVScraper:
                                     self.play_vv_items.remove(existing_item)
                                     self.play_vv_items.append(item_data)
                                 else:
-                                    # 已有数据更好，跳过
+                                    # 已有数据更好，跳过当前数据但继续递归解析其他数据
                                     logging.info(f'⏭️ 跳过重复短剧: {mix_name} (当前: {vv:,}, 已有: {existing_vv:,})')
-                                    return  # 跳过当前数据
+                                    # 注意：不使用return，避免中断递归解析
                             else:
                                 # 不存在，直接添加
                                 self.play_vv_items.append(item_data)
@@ -1049,14 +1053,20 @@ class DouyinPlayVVScraper:
                 vv = int(match.group(3))
                 episodes = int(match.group(4))
                 
-                # 数据验证：确保播放量大于0且合集名称不为空
+                # 数据验证：确保有mix_id（按短剧ID去重）
+                # 注意：播放量为0的数据也会被保存，可能是新发布的短剧
                 if vv <= 0:
-                    logging.warning(f"正则提取跳过无效的播放量数据: mix_name={mix_name}, play_vv={vv}")
+                    logging.warning(f"⚠️ 发现播放量为0的数据: mix_name={mix_name}, play_vv={vv}，仍会保存")
+                
+                # 检查mix_id，如果没有则跳过
+                if not mix_id or mix_id.strip() == "":
+                    logging.warning(f"正则提取跳过缺少mix_id的数据: play_vv={vv}, mix_name={mix_name}")
                     continue
                 
+                # 如果mix_name为空，使用mix_id作为名称
                 if not mix_name or mix_name.strip() == "":
-                    logging.warning(f"正则提取跳过缺少合集名称的数据: play_vv={vv}")
+                    mix_name = f"短剧_{mix_id}"
-                    continue
+                    logging.warning(f"⚠️ mix_name为空，使用mix_id作为名称: {mix_name}")
                 
                 # 构建合集链接
                 video_url = f"https://www.douyin.com/collection/{mix_id}" if mix_id else ""
@@ -1088,10 +1098,9 @@ class DouyinPlayVVScraper:
         for match in re.findall(r'"play_vv"\s*:\s*(\d+)', text):
             try:
                 vv = int(match)
-                # 数据验证：跳过无效的播放量数据
+                # 数据验证：播放量为0的数据也会被保存
                 if vv <= 0:
-                    logging.warning(f"跳过无效的播放量数据: play_vv={vv}")
+                    logging.warning(f"⚠️ 发现播放量为0的数据: play_vv={vv}，仍会保存")
-                    continue
                     
                 # 检查是否已经存在相同的play_vv
                 if not any(item['play_vv'] == vv for item in self.play_vv_items):
@@ -1208,10 +1217,9 @@ class DouyinPlayVVScraper:
             for m in re.findall(r'"statis"\s*:\s*\{[^}]*"play_vv"\s*:\s*(\d+)[^}]*\}', page_source):
                 try:
                     vv = int(m)
-                    # 数据验证：跳过无效的播放量数据
+                    # 数据验证：播放量为0的数据也会被保存
                     if vv <= 0:
-                        logging.warning(f"跳过无效的播放量数据: play_vv={vv}")
+                        logging.warning(f"⚠️ 发现播放量为0的数据: play_vv={vv}，仍会保存")
-                        continue
                         
                     # 检查是否已经存在相同的play_vv
                     if not any(item['play_vv'] == vv for item in self.play_vv_items):

frontend/src/AdminPanel.vue

@@ -8,9 +8,7 @@ const router = useRouter()
 // 响应式数据
 const rankingData = ref([])
 const loading = ref(false)
-const selectedDate = ref('')
 const showEditModal = ref(false)
-const showAddModal = ref(false)
 
 // 编辑表单数据
 const editForm = reactive({
@@ -34,32 +32,20 @@ const editForm = reactive({
   isDrama: false
 })
 
-// 新增表单数据
-const addForm = reactive({
-  title: '',
-  mix_name: '',
-  series_author: '',
-  Manufacturing_Field: '',
-  Copyright_field: '',
-  play_vv: 0,
-  total_likes_formatted: '',
-  cover_image_url: '',
-  cover_backup_urls: [],
-  timeline_data: {
-    play_vv_change: 0,
-    play_vv_change_rate: 0
-  }
-})
-
-// 初始化日期为今天
-const initDate = () => {
-  const today = new Date()
-  selectedDate.value = today.toISOString().split('T')[0]
-}
-
 // API基础URL
 const API_BASE_URL = 'http://159.75.150.210:8443/api'
 
+// 格式化播放量
+const formatPlayCount = (count) => {
+  if (!count) return '0'
+  if (count >= 100000000) {
+    return (count / 100000000).toFixed(1) + '亿'
+  } else if (count >= 10000) {
+    return (count / 10000).toFixed(1) + '万'
+  }
+  return count.toString()
+}
+
 // 获取排行榜数据
 const fetchRankingData = async () => {
   loading.value = true
@@ -68,9 +54,7 @@ const fetchRankingData = async () => {
       params: {
         page: 1,
         limit: 100,
-        sort: 'growth',
+        sort: 'growth'
-        start_date: selectedDate.value,
-        end_date: selectedDate.value
       }
     })
     
@@ -119,17 +103,6 @@ const fetchRankingData = async () => {
   }
 }
 
-// 格式化播放量
-const formatPlayCount = (count) => {
-  if (!count) return '0'
-  if (count >= 100000000) {
-    return (count / 100000000).toFixed(1) + '亿'
-  } else if (count >= 10000) {
-    return (count / 10000).toFixed(1) + '万'
-  }
-  return count.toString()
-}
-
 // 编辑项目
 const editItem = async (item) => {
   editForm.id = item.id || item._id
@@ -273,8 +246,7 @@ const saveEdit = async () => {
       total_likes_formatted: editForm.total_likes_formatted,
       cover_image_url: editForm.cover_image_url,
       cover_backup_urls: editForm.cover_backup_urls,
-      timeline_data: editForm.timeline_data,
+      timeline_data: editForm.timeline_data
-      target_date: selectedDate.value
     }
     
     // 调用后端API更新数据
@@ -312,113 +284,11 @@ const saveEdit = async () => {
   }
 }
 
-// 添加新项目
-const addNewItem = async () => {
-  try {
-    const newItemData = {
-      title: addForm.title,
-      mix_name: addForm.mix_name,
-      series_author: addForm.series_author,
-      Manufacturing_Field: addForm.Manufacturing_Field,
-      Copyright_field: addForm.Copyright_field,
-      play_vv: addForm.play_vv,
-      total_likes_formatted: addForm.total_likes_formatted,
-      cover_image_url: addForm.cover_image_url,
-      cover_backup_urls: addForm.cover_backup_urls,
-      timeline_data: addForm.timeline_data
-    }
-    
-    // 尝试调用后端API添加数据
-    try {
-      const response = await axios.post(`${API_BASE_URL}/rank/videos`, newItemData)
-      
-      // 重置表单
-      Object.keys(addForm).forEach(key => {
-        if (key === 'play_vv') {
-          addForm[key] = 0
-        } else if (key === 'cover_backup_urls') {
-          addForm[key] = []
-        } else if (key === 'timeline_data') {
-          addForm[key] = {
-            play_vv_change: 0,
-            play_vv_change_rate: 0
-          }
-        } else {
-          addForm[key] = ''
-        }
-      })
-      
-      showAddModal.value = false
-      
-      // 重新获取最新数据，确保前端显示的是数据库中的最新数据
-      await fetchRankingData()
-      
-      alert('添加成功！')
-    } catch (apiError) {
-      console.warn('API添加失败，使用本地添加:', apiError)
-      const newItem = {
-        id: Date.now(),
-        ...newItemData
-      }
-      
-      // 添加到本地数据
-      rankingData.value.unshift(newItem)
-      
-      // 重置表单
-      Object.keys(addForm).forEach(key => {
-        if (key === 'play_vv') {
-          addForm[key] = 0
-        } else if (key === 'cover_backup_urls') {
-          addForm[key] = []
-        } else if (key === 'timeline_data') {
-          addForm[key] = {
-            play_vv_change: 0,
-            play_vv_change_rate: 0
-          }
-        } else {
-          addForm[key] = ''
-        }
-      })
-      
-      showAddModal.value = false
-      alert('添加失败，但本地数据已更新。请检查网络连接。')
-    }
-  } catch (error) {
-    console.error('添加失败:', error)
-    alert('添加失败！')
-  }
-}
-
 // 取消编辑
 const cancelEdit = () => {
   showEditModal.value = false
 }
 
-// 取消添加
-const cancelAdd = () => {
-  showAddModal.value = false
-  // 重置表单
-  Object.keys(addForm).forEach(key => {
-    if (key === 'play_vv') {
-      addForm[key] = 0
-    } else if (key === 'cover_backup_urls') {
-      addForm[key] = []
-    } else if (key === 'timeline_data') {
-      addForm[key] = {
-        play_vv_change: 0,
-        play_vv_change_rate: 0
-      }
-    } else {
-      addForm[key] = ''
-    }
-  })
-}
-
-// 日期改变处理
-const onDateChange = () => {
-  fetchRankingData()
-}
-
 // 返回前端页面
 const goBack = () => {
   router.push('/')
@@ -426,7 +296,6 @@ const goBack = () => {
 
 // 页面加载时初始化
 onMounted(() => {
-  initDate()
   fetchRankingData()
 })
 </script>
@@ -442,28 +311,12 @@ onMounted(() => {
           <h1 class="main-title">AI棒榜 - 后台管理</h1>
         </div>
         <div class="header-actions">
-          <button class="btn btn-primary" @click="showAddModal = true">
-            添加新项目
-          </button>
           <button class="btn btn-secondary" @click="fetchRankingData">
             刷新数据
           </button>
         </div>
       </div>
 
-      <!-- 日期选择区域 -->
-      <div class="date-section">
-        <label for="date-input">选择日期：</label>
-        <input 
-          id="date-input"
-          type="date" 
-          v-model="selectedDate" 
-          @change="onDateChange"
-          class="date-input"
-        />
-        <span class="data-count">共 {{ rankingData.length }} 条数据</span>
-      </div>
-
       <!-- 管理内容区域 -->
       <div class="admin-content">
         <!-- 加载状态 -->
@@ -624,58 +477,6 @@ onMounted(() => {
         </div>
       </div>
     </div>
-
-    <!-- 添加模态框 -->
-    <div v-if="showAddModal" class="modal-overlay" @click="cancelAdd">
-      <div class="modal-content" @click.stop>
-        <div class="modal-header">
-          <h3>添加新项目</h3>
-          <button class="close-btn" @click="cancelAdd">×</button>
-        </div>
-        <div class="modal-body">
-          <div class="form-group">
-            <label>剧名：</label>
-            <input v-model="addForm.title" type="text" class="form-input" />
-          </div>
-          <div class="form-group">
-            <label>合集名：</label>
-            <input v-model="addForm.mix_name" type="text" class="form-input" />
-          </div>
-          <div class="form-group">
-            <label>剧场名：</label>
-            <input v-model="addForm.series_author" type="text" class="form-input" />
-          </div>
-          <div class="form-group">
-            <label>承制方：</label>
-            <input v-model="addForm.Manufacturing_Field" type="text" class="form-input" />
-          </div>
-          <div class="form-group">
-            <label>版权方：</label>
-            <input v-model="addForm.Copyright_field" type="text" class="form-input" />
-          </div>
-          <div class="form-group">
-            <label>播放量：</label>
-            <input v-model.number="addForm.play_vv" type="number" class="form-input" />
-          </div>
-          <div class="form-group">
-            <label>点赞数（格式化）：</label>
-            <input v-model="addForm.total_likes_formatted" type="text" class="form-input" />
-          </div>
-          <div class="form-group">
-            <label>封面图片URL：</label>
-            <input v-model="addForm.cover_image_url" type="url" class="form-input" />
-          </div>
-          <div class="form-group">
-            <label>增长量：</label>
-            <input v-model.number="addForm.timeline_data.play_vv_change" type="number" class="form-input" />
-          </div>
-        </div>
-        <div class="modal-footer">
-          <button class="btn btn-secondary" @click="cancelAdd">取消</button>
-          <button class="btn btn-primary" @click="addNewItem">添加</button>
-        </div>
-      </div>
-    </div>
   </div>
 </template>
 
@@ -807,29 +608,6 @@ export default {
   background: #c82333;
 }
 
-/* 日期选择区域 */
-.date-section {
-  padding: 16px;
-  background: white;
-  border-bottom: 1px solid #e0e0e0;
-  display: flex;
-  align-items: center;
-  gap: 12px;
-  font-size: 14px;
-}
-
-.date-input {
-  padding: 6px 10px;
-  border: 1px solid #ddd;
-  border-radius: 4px;
-  font-size: 14px;
-}
-
-.data-count {
-  color: #666;
-  margin-left: auto;
-}
-
 /* 管理内容区域 */
 .admin-content {
   padding: 16px;
@@ -1158,10 +936,6 @@ export default {
     padding: 16px 12px;
   }
   
-  .date-section {
-    padding: 12px;
-  }
-  
   .admin-content {
     padding: 12px;
   }