refactor(rumble): un seul cœur de scraping, routeur fin, contrat 503 CF
CI / build-and-test (push) Successful in 14m57s
CI / build-and-test (push) Successful in 14m57s
- server/providers/rumble.mjs devient l'unique implémentation : fetch (cookie jar __cf_bm + curl_cffi + cooldown global 60 s après un échec total), parsing liste/vidéo avec surcouche JSON-LD partagée, vues via parseRumbleViews partout (« 1,2K » n'est plus lu en 12 sur /api/rumble). - server/rumble.mjs = couche HTTP mince : limiter 20/min, cache 60 s, 5 routes qui délèguent au cœur ; /search passe par le registre de providers (cache SQLite, negative cache et channelRef partagés avec /api/search). - Contrat d'erreur : un blocage Cloudflare est un 503 rumble_cloudflare_challenge, plus jamais un 404 (qui faisait retirer les vidéos du flux Shorts lors d'un challenge) ; page reçue sans identité = 404 rumble_video_not_found. - Code mort supprimé : scrapeRumbleVideo d'index.mjs (191 l., 3e implémentation jamais appelée), rumbleLimiter 10/min jamais monté, import cheerio devenu inutile. Net -238 lignes. - Tests : parseur générique (vues, durées, ids /shorts/), contrat 503 sans réseau via le cooldown, durées absentes -> undefined. - Docs : API_MCP_GUIDE §5.5 (routes, rate-limit réel, erreurs) + note README « Rumble peut être temporairement absent selon le réseau ».
This commit is contained in:
@@ -12,7 +12,6 @@ import { execFile as execFileCb } from 'node:child_process';
|
||||
import { promisify } from 'node:util';
|
||||
import { fileURLToPath as serverFileURLToPath } from 'node:url';
|
||||
import ffmpegPath from 'ffmpeg-static';
|
||||
import * as cheerio from 'cheerio';
|
||||
import axios from 'axios';
|
||||
import rumbleRouter from './rumble.mjs';
|
||||
import { providerRegistry, getProviderAdapter, validateProviders, SUGGESTION_CONTRACT_VERSION } from './providers/registry.mjs';
|
||||
@@ -1140,15 +1139,6 @@ function formatsCacheSet(key, data) {
|
||||
}
|
||||
}
|
||||
|
||||
// Rate limiter for Rumble scraping to prevent being blocked
|
||||
const rumbleLimiter = rateLimit({
|
||||
windowMs: 60 * 1000, // 1 min
|
||||
max: 10, // Limit to 10 requests per minute per IP
|
||||
standardHeaders: true,
|
||||
legacyHeaders: false,
|
||||
message: { error: 'Too many requests to Rumble API. Please try again later.' }
|
||||
});
|
||||
|
||||
function makeAccessToken(userId, sessionId) {
|
||||
const payload = { sub: userId, sid: sessionId };
|
||||
return jwt.sign(payload, JWT_SECRET, { expiresIn: `${ACCESS_TTL_MIN}m` });
|
||||
@@ -1300,198 +1290,6 @@ function providerUrlFrom(provider, videoId, { instance, slug, sourceUrl }) {
|
||||
}
|
||||
}
|
||||
|
||||
async function scrapeRumbleVideo(videoId) {
|
||||
const url = `https://rumble.com/${videoId}`;
|
||||
|
||||
try {
|
||||
const response = await axios.get(url, {
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
||||
'Accept-Language': 'en-US,en;q=0.5',
|
||||
'Accept-Encoding': 'gzip, deflate, br',
|
||||
'DNT': '1',
|
||||
'Connection': 'keep-alive',
|
||||
'Upgrade-Insecure-Requests': '1',
|
||||
'Cache-Control': 'no-cache'
|
||||
},
|
||||
timeout: 15000,
|
||||
maxRedirects: 5,
|
||||
validateStatus: function (status) {
|
||||
return status >= 200 && status < 400; // Accept redirects
|
||||
}
|
||||
});
|
||||
|
||||
const $ = cheerio.load(response.data);
|
||||
const html = response.data;
|
||||
|
||||
// Extract basic video information
|
||||
let title = '';
|
||||
let thumbnail = '';
|
||||
let uploaderName = '';
|
||||
let uploaderAvatar = '';
|
||||
let views = 0;
|
||||
let duration = 0;
|
||||
let uploadedDate = '';
|
||||
let description = '';
|
||||
|
||||
// Try multiple selectors for title (Rumble's HTML structure can vary)
|
||||
title = $('h1.video-title, .video-title h1, [data-video-title]').first().text().trim() ||
|
||||
$('meta[property="og:title"]').attr('content') ||
|
||||
$('title').text().trim() ||
|
||||
$('h1').first().text().trim() || '';
|
||||
|
||||
// Clean up title (remove site name if present)
|
||||
title = title.replace(/\s*\|\s*Rumble$/i, '').trim();
|
||||
|
||||
// Extract thumbnail with fallbacks
|
||||
thumbnail = $('meta[property="og:image"], meta[name="twitter:image"]').attr('content') ||
|
||||
$('meta[property="og:image:secure_url"]').attr('content') ||
|
||||
$('.video-thumbnail img, .thumbnail img').attr('src') || '';
|
||||
|
||||
// Make thumbnail URL absolute if relative
|
||||
if (thumbnail && !thumbnail.startsWith('http')) {
|
||||
thumbnail = thumbnail.startsWith('//') ? 'https:' + thumbnail : 'https://rumble.com' + thumbnail;
|
||||
}
|
||||
|
||||
// Extract uploader information with multiple selectors
|
||||
uploaderName = $('.media-by--a, .channel-name, .uploader-name').first().text().trim() ||
|
||||
$('meta[property="article:author"]').attr('content') ||
|
||||
$('.author-name, .channel-link').first().text().trim() || '';
|
||||
|
||||
uploaderAvatar = $('.channel-avatar img, .uploader-avatar img').attr('src') ||
|
||||
$('.author-avatar img').attr('src') || '';
|
||||
|
||||
// Make uploader avatar URL absolute
|
||||
if (uploaderAvatar && !uploaderAvatar.startsWith('http')) {
|
||||
uploaderAvatar = uploaderAvatar.startsWith('//') ? 'https:' + uploaderAvatar : 'https://rumble.com' + uploaderAvatar;
|
||||
}
|
||||
|
||||
// Extract views with better parsing
|
||||
const viewsText = $('.rumbles-views, .video-views, .views-count, .video-info .views').first().text().trim();
|
||||
if (viewsText) {
|
||||
const viewsMatch = viewsText.match(/([\d,]+(?:\.\d+)?)\s*(K|M|B)?/i);
|
||||
if (viewsMatch) {
|
||||
let num = parseFloat(viewsMatch[1].replace(/,/g, ''));
|
||||
const multiplier = viewsMatch[2]?.toUpperCase();
|
||||
if (multiplier === 'K') num *= 1000;
|
||||
else if (multiplier === 'M') num *= 1000000;
|
||||
else if (multiplier === 'B') num *= 1000000000;
|
||||
views = Math.floor(num);
|
||||
} else {
|
||||
// Try direct number parsing
|
||||
const directMatch = viewsText.match(/(\d+(?:,\d+)*)/);
|
||||
if (directMatch) {
|
||||
views = parseInt(directMatch[1].replace(/,/g, ''));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Extract duration with improved parsing
|
||||
const durationText = $('meta[property="video:duration"]').attr('content') ||
|
||||
$('video').attr('duration') ||
|
||||
$('.video-duration, .duration, .video-time').first().text().trim() ||
|
||||
$('.time-duration').text().trim();
|
||||
|
||||
if (durationText) {
|
||||
if (!isNaN(durationText)) {
|
||||
duration = parseInt(durationText);
|
||||
} else {
|
||||
// Parse various duration formats
|
||||
const timeMatch = durationText.match(/(\d+):(\d+)(?::(\d+))?/);
|
||||
if (timeMatch) {
|
||||
const hours = parseInt(timeMatch[3] || '0');
|
||||
const minutes = parseInt(timeMatch[1]);
|
||||
const seconds = parseInt(timeMatch[2]);
|
||||
duration = hours * 3600 + minutes * 60 + seconds;
|
||||
} else {
|
||||
// Try HH:MM:SS format or MM:SS
|
||||
const parts = durationText.split(':').map(p => parseInt(p.trim()) || 0);
|
||||
if (parts.length === 3) {
|
||||
duration = parts[0] * 3600 + parts[1] * 60 + parts[2];
|
||||
} else if (parts.length === 2) {
|
||||
duration = parts[0] * 60 + parts[1];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Extract upload date
|
||||
uploadedDate = $('meta[property="article:published_time"]').attr('content') ||
|
||||
$('.upload-date, .published-date, .video-date').first().text().trim() || '';
|
||||
|
||||
// Try to parse relative dates
|
||||
if (!uploadedDate || uploadedDate.includes('ago')) {
|
||||
const relativeDate = $('.upload-date, .published-date').first().text().trim();
|
||||
if (relativeDate && relativeDate.includes('ago')) {
|
||||
// Convert relative date to ISO string (simple conversion)
|
||||
const now = new Date();
|
||||
if (relativeDate.includes('hour')) {
|
||||
const hours = parseInt(relativeDate.match(/(\d+)/)?.[1] || '1');
|
||||
now.setHours(now.getHours() - hours);
|
||||
uploadedDate = now.toISOString();
|
||||
} else if (relativeDate.includes('day')) {
|
||||
const days = parseInt(relativeDate.match(/(\d+)/)?.[1] || '1');
|
||||
now.setDate(now.getDate() - days);
|
||||
uploadedDate = now.toISOString();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Extract description
|
||||
description = $('meta[property="og:description"]').attr('content') ||
|
||||
$('.video-description, .description, .video-summary').first().text().trim() || '';
|
||||
|
||||
// Extract video ID from various sources
|
||||
let extractedVideoId = videoId;
|
||||
const videoIdMatch = html.match(/"video_id"\s*:\s*"([^"]+)"/) ||
|
||||
html.match(/video[_-]id["\s:]+([^"\s]+)/) ||
|
||||
html.match(/embed\/([^/?]+)/);
|
||||
if (videoIdMatch && videoIdMatch[1]) {
|
||||
extractedVideoId = videoIdMatch[1];
|
||||
}
|
||||
|
||||
// Validate extracted data
|
||||
const isValidVideo = title || thumbnail || uploaderName;
|
||||
|
||||
return {
|
||||
videoId: extractedVideoId,
|
||||
title: title || 'Untitled Video',
|
||||
thumbnail,
|
||||
uploaderName: uploaderName || 'Unknown Uploader',
|
||||
uploaderAvatar: uploaderAvatar || thumbnail,
|
||||
views: Math.max(0, views),
|
||||
duration: Math.max(0, duration),
|
||||
uploadedDate: uploadedDate || new Date().toISOString(),
|
||||
description,
|
||||
url,
|
||||
type: 'video',
|
||||
scraped: true,
|
||||
confidence: isValidVideo ? 'high' : 'low'
|
||||
};
|
||||
} catch (error) {
|
||||
console.error('Erreur scraping Rumble:', error.message);
|
||||
|
||||
// Return minimal data for fallback with error info
|
||||
return {
|
||||
videoId,
|
||||
title: 'Video unavailable',
|
||||
thumbnail: '',
|
||||
uploaderName: 'Unknown',
|
||||
uploaderAvatar: '',
|
||||
views: 0,
|
||||
duration: 0,
|
||||
uploadedDate: '',
|
||||
description: '',
|
||||
url,
|
||||
type: 'video',
|
||||
error: error.message,
|
||||
scraped: false,
|
||||
confidence: 'none'
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function guessContentTypeByExt(ext) {
|
||||
const e = String(ext || '').toLowerCase();
|
||||
if (e === 'mp4' || e === 'm4v') return 'video/mp4';
|
||||
|
||||
Reference in New Issue
Block a user