优化增强了获取评论内容机制，
现在抓取评论内容的方法是网络日志拦截抖音API的方法， 这个获取的评论内容是随机的，但是里面添加了去重的机制， 针对于过少的评论和无评论做了优化， 添加监控超时限制，爬取评论内容一条视频监控超时时间为5小时（因为有的视频评论数量太大）
This commit is contained in:
parent
64455034bb
commit
b26852b95f
@ -79,7 +79,6 @@ class DouyinPlayVVScraper:
|
||||
self.db = None
|
||||
self.collection = None
|
||||
self.image_cache = {} # 图片ID到TOS链接的缓存映射 {image_id: tos_url}
|
||||
self.all_collected_comments = [] # 存储所有收集到的评论数据
|
||||
|
||||
# 实时存储相关属性
|
||||
self.batch_id = str(uuid.uuid4()) # 每次运行的唯一标识
|
||||
@ -1771,6 +1770,12 @@ class DouyinPlayVVScraper:
|
||||
else:
|
||||
logging.warning(f'[增量更新] 第 {i} 集视频详细数据获取失败: {mix_name}')
|
||||
|
||||
# 添加随机延迟避免请求过快，模拟人类行为
|
||||
if i < len(episode_video_ids): # 不是最后一个视频时才延迟
|
||||
random_delay = self.anti_detection.get_human_like_delay()
|
||||
logging.info(f'🕐 [增量更新] 视频间隔等待时间: {random_delay:.1f}秒')
|
||||
time.sleep(random_delay)
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f'[增量更新] 处理第 {i} 集视频时出错: {mix_name} - {e}')
|
||||
continue
|
||||
@ -2037,8 +2042,8 @@ class DouyinPlayVVScraper:
|
||||
# 提交滑动任务
|
||||
scroll_future = executor.submit(self._async_scroll_task_with_state, max_scroll_attempts, scroll_delay, shared_state)
|
||||
|
||||
# 同时提交监控任务 - 监控任务会检测滑动任务状态
|
||||
monitor_future = executor.submit(self._async_monitor_task_with_state, video_id, collected_comment_ids, shared_state, 3600,
|
||||
# 同时提交监控任务 - 监控任务会检测滑动任务状态（5小时超时）
|
||||
monitor_future = executor.submit(self._async_monitor_task_with_state, video_id, collected_comment_ids, shared_state, 18000,
|
||||
document_id, episode_number, mix_name)
|
||||
|
||||
# 等待两个任务完成
|
||||
@ -2049,10 +2054,90 @@ class DouyinPlayVVScraper:
|
||||
|
||||
logging.info(f'评论滑动加载完成，共收集到 {len(all_comments)} 条评论')
|
||||
|
||||
# 针对评论较少的情况，执行补丁机制确保不遗漏评论
|
||||
# 当滑动次数较少（可能只滑动了2-3次就到底）但评论数量也较少时，
|
||||
# 很可能存在页面上可见但未被网络日志捕获的评论
|
||||
|
||||
# 智能判断是否需要执行补丁机制
|
||||
should_apply_patch = False
|
||||
|
||||
# 条件1: 评论数量较少（少于20条）
|
||||
if len(all_comments) < 20:
|
||||
should_apply_patch = True
|
||||
logging.debug(f'评论数量较少({len(all_comments)}条)，启用补丁机制')
|
||||
|
||||
# 条件2: 检查页面上是否有更多可见的评论元素
|
||||
try:
|
||||
visible_comment_count = self.driver.execute_script("""
|
||||
var selectors = [
|
||||
'[data-e2e="comment-item"]',
|
||||
'[class*="comment-item"]',
|
||||
'[class*="comment-content"]'
|
||||
];
|
||||
var totalCount = 0;
|
||||
selectors.forEach(function(selector) {
|
||||
var elements = document.querySelectorAll(selector);
|
||||
elements.forEach(function(element) {
|
||||
if (element.offsetParent !== null && element.textContent.trim().length > 2) {
|
||||
totalCount++;
|
||||
}
|
||||
});
|
||||
});
|
||||
return totalCount;
|
||||
""")
|
||||
|
||||
if visible_comment_count > len(all_comments):
|
||||
should_apply_patch = True
|
||||
logging.debug(f'页面可见评论({visible_comment_count}条) > 已获取评论({len(all_comments)}条)，启用补丁机制')
|
||||
|
||||
except Exception as e:
|
||||
logging.debug(f'检查页面可见评论数量失败: {e}')
|
||||
# 如果检查失败，对于少量评论的情况仍然执行补丁
|
||||
if len(all_comments) < 10:
|
||||
should_apply_patch = True
|
||||
|
||||
patch_comments = []
|
||||
if should_apply_patch:
|
||||
logging.info('执行评论补丁机制...')
|
||||
patch_comments = self._extract_comments_patch(video_id)
|
||||
else:
|
||||
logging.debug('无需执行补丁机制')
|
||||
|
||||
if patch_comments:
|
||||
# 去重合并补丁评论
|
||||
existing_texts = {comment.get('text', '') for comment in all_comments}
|
||||
new_patch_comments = []
|
||||
|
||||
for patch_comment in patch_comments:
|
||||
if patch_comment.get('text', '') not in existing_texts:
|
||||
new_patch_comments.append(patch_comment)
|
||||
existing_texts.add(patch_comment.get('text', ''))
|
||||
|
||||
if new_patch_comments:
|
||||
all_comments.extend(new_patch_comments)
|
||||
logging.info(f'补丁机制额外获取到 {len(new_patch_comments)} 条评论，总计 {len(all_comments)} 条评论')
|
||||
|
||||
# 如果有新的评论且启用了实时保存，更新数据库
|
||||
if document_id and episode_number and new_patch_comments:
|
||||
try:
|
||||
self.update_video_comments_realtime(document_id, episode_number, new_patch_comments, mix_name)
|
||||
logging.info(f'实时保存补丁评论到数据库: {len(new_patch_comments)} 条')
|
||||
except Exception as e:
|
||||
logging.warning(f'实时保存补丁评论失败: {e}')
|
||||
else:
|
||||
logging.debug('补丁机制未发现新的评论')
|
||||
else:
|
||||
logging.debug('补丁机制未获取到任何评论')
|
||||
|
||||
# 保存评论到文件
|
||||
if all_comments:
|
||||
self.save_comments_to_file(all_comments, video_id)
|
||||
|
||||
# 添加随机停留时间，防止网页被爬取崩溃
|
||||
rest_time = random.uniform(10, 20) # 10-20秒随机停留
|
||||
logging.info(f'评论抓取完成，停留 {rest_time:.1f} 秒以保护网页稳定性...')
|
||||
time.sleep(rest_time)
|
||||
|
||||
return all_comments
|
||||
|
||||
except Exception as e:
|
||||
@ -3099,8 +3184,10 @@ class DouyinPlayVVScraper:
|
||||
video_details['episode_number'] = i
|
||||
video_details_list.append(video_details)
|
||||
|
||||
# 添加延迟避免请求过快
|
||||
time.sleep(2)
|
||||
# 添加随机延迟避免请求过快，模拟人类行为
|
||||
random_delay = self.anti_detection.get_human_like_delay()
|
||||
logging.info(f'🕐 视频间隔等待时间: {random_delay:.1f}秒')
|
||||
time.sleep(random_delay)
|
||||
# exit(0)
|
||||
|
||||
except Exception as e:
|
||||
@ -3132,6 +3219,297 @@ class DouyinPlayVVScraper:
|
||||
self.cookies = {cookie['name']: cookie['value'] for cookie in self.driver.get_cookies()}
|
||||
return self.cookies
|
||||
|
||||
def _extract_comments_patch(self, video_id: str) -> list:
|
||||
"""
|
||||
评论抓取补丁机制 - 强制从页面元素中提取所有可见评论
|
||||
针对评论较少但获取不全的情况，确保不遗漏任何页面上可见的评论
|
||||
"""
|
||||
comments = []
|
||||
try:
|
||||
logging.info(f'开始执行评论抓取补丁机制，视频ID: {video_id}')
|
||||
|
||||
# 首先检查是否存在"抢首评"按钮，如果存在说明视频确实没有评论
|
||||
if self._check_first_comment_button():
|
||||
logging.info('检测到"抢首评"按钮，确认视频没有评论，跳过补丁机制')
|
||||
return comments
|
||||
|
||||
# 等待页面稳定
|
||||
time.sleep(1)
|
||||
|
||||
# 滚动到评论区域确保评论完全加载
|
||||
self._scroll_to_comment_section()
|
||||
time.sleep(1)
|
||||
|
||||
# 针对少量评论优化的选择器列表
|
||||
comment_selectors = [
|
||||
# 抖音常用的评论选择器（优先级最高）
|
||||
'[data-e2e="comment-item"]',
|
||||
'[data-e2e="comment-list"] > div',
|
||||
'[class*="comment-item"]',
|
||||
'[class*="comment-content"]',
|
||||
|
||||
# 针对少量评论的特殊选择器
|
||||
'[class*="comment"] [class*="content"]',
|
||||
'[class*="Comment"] [class*="content"]',
|
||||
'[class*="comment-text"]',
|
||||
'[class*="user-comment"]',
|
||||
|
||||
# 更通用的评论选择器
|
||||
'div[class*="comment"]:not([class*="input"]):not([class*="button"]):not([class*="header"])',
|
||||
'li[class*="comment"]',
|
||||
'[role="listitem"][class*="comment"]',
|
||||
|
||||
# 备用选择器（用于兜底）
|
||||
'.comment-list > div',
|
||||
'.comment-container > div',
|
||||
'.comment-wrapper > div',
|
||||
|
||||
# 针对少量评论的深度选择器
|
||||
'div[class*="comment"] p',
|
||||
'div[class*="comment"] span:not([class*="button"]):not([class*="icon"])',
|
||||
'[data-e2e*="comment"] div[class*="text"]'
|
||||
]
|
||||
|
||||
collected_texts = set() # 用于去重
|
||||
|
||||
for selector in comment_selectors:
|
||||
try:
|
||||
logging.debug(f'尝试选择器: {selector}')
|
||||
elements = self.driver.find_elements("css selector", selector)
|
||||
|
||||
if elements:
|
||||
logging.debug(f'选择器 {selector} 找到 {len(elements)} 个元素')
|
||||
|
||||
for element in elements:
|
||||
try:
|
||||
if not element.is_displayed():
|
||||
continue
|
||||
|
||||
# 获取评论文本
|
||||
comment_text = element.text.strip()
|
||||
|
||||
# 过滤无效文本
|
||||
if not comment_text or len(comment_text) < 2:
|
||||
continue
|
||||
|
||||
# 过滤系统文本和按钮文本
|
||||
skip_texts = [
|
||||
'回复', '点赞', '举报', '删除', '编辑', '查看更多',
|
||||
'展开', '收起', '暂时没有更多评论', '加载中',
|
||||
'发布', '取消', '确定', '登录', '注册', '关注'
|
||||
]
|
||||
|
||||
# 检查是否为纯按钮文本
|
||||
if comment_text in skip_texts:
|
||||
continue
|
||||
|
||||
# 清理包含按钮文本的评论
|
||||
original_text = comment_text
|
||||
for skip_text in skip_texts:
|
||||
comment_text = comment_text.replace(skip_text, '').strip()
|
||||
|
||||
# 如果清理后文本太短，跳过
|
||||
if len(comment_text) < 2:
|
||||
continue
|
||||
|
||||
# 去重检查
|
||||
if comment_text in collected_texts:
|
||||
continue
|
||||
|
||||
collected_texts.add(comment_text)
|
||||
|
||||
# 创建评论对象
|
||||
comment_info = {
|
||||
'text': comment_text,
|
||||
'user_name': '',
|
||||
'digg_count': 0,
|
||||
'create_time': 0,
|
||||
'source': f'patch_{selector}' # 标记来源
|
||||
}
|
||||
|
||||
comments.append(comment_info)
|
||||
logging.debug(f'补丁提取评论: {comment_text[:30]}...')
|
||||
|
||||
except Exception as e:
|
||||
logging.debug(f'处理评论元素失败: {e}')
|
||||
continue
|
||||
|
||||
except Exception as e:
|
||||
logging.debug(f'选择器 {selector} 执行失败: {e}')
|
||||
continue
|
||||
|
||||
# 使用JavaScript进一步搜索评论文本（针对少量评论优化）
|
||||
try:
|
||||
js_comments = self.driver.execute_script("""
|
||||
var comments = [];
|
||||
var processedTexts = new Set();
|
||||
|
||||
// 针对少量评论的优化选择器
|
||||
var selectors = [
|
||||
'div[class*="comment"]',
|
||||
'li[class*="comment"]',
|
||||
'[data-e2e*="comment"]',
|
||||
'[class*="comment-item"]',
|
||||
'[class*="comment-content"]'
|
||||
];
|
||||
|
||||
selectors.forEach(function(selector) {
|
||||
try {
|
||||
var elements = document.querySelectorAll(selector);
|
||||
elements.forEach(function(element) {
|
||||
if (element.offsetParent === null) return; // 跳过不可见元素
|
||||
|
||||
var text = element.textContent.trim();
|
||||
if (text.length < 2) return;
|
||||
|
||||
// 过滤按钮文本
|
||||
var skipTexts = ['回复', '点赞', '举报', '删除', '编辑', '查看更多', '展开', '收起', '暂时没有更多评论', '关注'];
|
||||
var isButtonText = skipTexts.some(function(skipText) {
|
||||
return text === skipText;
|
||||
});
|
||||
|
||||
if (isButtonText || processedTexts.has(text)) return;
|
||||
|
||||
// 清理文本
|
||||
var cleanText = text;
|
||||
skipTexts.forEach(function(skipText) {
|
||||
cleanText = cleanText.replace(new RegExp(skipText, 'g'), '').trim();
|
||||
});
|
||||
|
||||
if (cleanText.length >= 2 && !processedTexts.has(cleanText)) {
|
||||
processedTexts.add(cleanText);
|
||||
comments.push({
|
||||
text: cleanText,
|
||||
user_name: '',
|
||||
digg_count: 0,
|
||||
create_time: 0,
|
||||
source: 'patch_js'
|
||||
});
|
||||
}
|
||||
});
|
||||
} catch (e) {
|
||||
console.log('JS选择器执行失败:', selector, e);
|
||||
}
|
||||
});
|
||||
|
||||
return comments;
|
||||
""")
|
||||
|
||||
if js_comments:
|
||||
# 去重合并
|
||||
existing_texts = {comment['text'] for comment in comments}
|
||||
for js_comment in js_comments:
|
||||
if js_comment['text'] not in existing_texts:
|
||||
comments.append(js_comment)
|
||||
existing_texts.add(js_comment['text'])
|
||||
|
||||
logging.debug(f'JavaScript补丁额外提取 {len(js_comments)} 条评论')
|
||||
|
||||
except Exception as e:
|
||||
logging.debug(f'JavaScript评论提取失败: {e}')
|
||||
|
||||
if comments:
|
||||
logging.info(f'评论抓取补丁机制成功提取 {len(comments)} 条评论')
|
||||
else:
|
||||
logging.warning('评论抓取补丁机制未找到任何评论')
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f'评论抓取补丁机制执行失败: {e}')
|
||||
|
||||
return comments
|
||||
|
||||
def _check_first_comment_button(self) -> bool:
|
||||
"""
|
||||
检测是否存在"抢首评"按钮，如果存在说明视频确实没有评论
|
||||
Returns:
|
||||
bool: True表示检测到抢首评按钮（视频没有评论），False表示没有检测到
|
||||
"""
|
||||
try:
|
||||
# 常见的"抢首评"相关文本
|
||||
first_comment_indicators = [
|
||||
"抢首评", "首评"
|
||||
]
|
||||
|
||||
logging.debug('检测"抢首评"按钮...')
|
||||
|
||||
# 方法1: 通过文本内容检测
|
||||
for indicator in first_comment_indicators:
|
||||
try:
|
||||
# 使用XPath查找包含指定文本的元素
|
||||
xpath_selectors = [
|
||||
f"//*[contains(text(), '{indicator}')]",
|
||||
f"//button[contains(text(), '{indicator}')]",
|
||||
f"//div[contains(text(), '{indicator}')]",
|
||||
f"//span[contains(text(), '{indicator}')]"
|
||||
]
|
||||
|
||||
for xpath in xpath_selectors:
|
||||
elements = self.driver.find_elements("xpath", xpath)
|
||||
if elements:
|
||||
for element in elements:
|
||||
try:
|
||||
if element.is_displayed() and element.text.strip():
|
||||
logging.debug(f'检测到抢首评相关文本: "{element.text.strip()}"')
|
||||
return True
|
||||
except Exception:
|
||||
continue
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
# 方法2: 通过CSS选择器检测评论输入框的占位符文本
|
||||
try:
|
||||
comment_input_selectors = [
|
||||
'input[placeholder*="抢首评"]',
|
||||
'textarea[placeholder*="抢首评"]',
|
||||
'[data-e2e="comment-input"]',
|
||||
'[class*="comment-input"]'
|
||||
]
|
||||
|
||||
for selector in comment_input_selectors:
|
||||
elements = self.driver.find_elements("css selector", selector)
|
||||
if elements:
|
||||
for element in elements:
|
||||
try:
|
||||
placeholder = element.get_attribute('placeholder') or ''
|
||||
if any(indicator in placeholder for indicator in first_comment_indicators):
|
||||
logging.debug(f'检测到抢首评输入框占位符: "{placeholder}"')
|
||||
return True
|
||||
except Exception:
|
||||
continue
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 方法3: 检测评论区域是否显示空状态
|
||||
try:
|
||||
empty_comment_selectors = [
|
||||
'[class*="empty"]',
|
||||
'[class*="no-comment"]',
|
||||
'[class*="comment-empty"]',
|
||||
'[data-e2e="comment-empty"]'
|
||||
]
|
||||
|
||||
for selector in empty_comment_selectors:
|
||||
elements = self.driver.find_elements("css selector", selector)
|
||||
if elements:
|
||||
for element in elements:
|
||||
try:
|
||||
if element.is_displayed():
|
||||
text = element.text.strip()
|
||||
if any(indicator in text for indicator in first_comment_indicators):
|
||||
logging.debug(f'检测到空评论状态: "{text}"')
|
||||
return True
|
||||
except Exception:
|
||||
continue
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
logging.debug('未检测到抢首评按钮或相关标识')
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logging.debug(f'检测抢首评按钮时出错: {e}')
|
||||
return False
|
||||
|
||||
def run(self):
|
||||
try:
|
||||
self.setup_driver()
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user