refactor(rumble): un seul cœur de scraping, routeur fin, contrat 503 CF
CI / build-and-test (push) Successful in 14m57s

- server/providers/rumble.mjs devient l'unique implémentation : fetch
  (cookie jar __cf_bm + curl_cffi + cooldown global 60 s après un échec
  total), parsing liste/vidéo avec surcouche JSON-LD partagée, vues via
  parseRumbleViews partout (« 1,2K » n'est plus lu en 12 sur /api/rumble).
- server/rumble.mjs = couche HTTP mince : limiter 20/min, cache 60 s,
  5 routes qui délèguent au cœur ; /search passe par le registre de
  providers (cache SQLite, negative cache et channelRef partagés avec
  /api/search).
- Contrat d'erreur : un blocage Cloudflare est un 503
  rumble_cloudflare_challenge, plus jamais un 404 (qui faisait retirer
  les vidéos du flux Shorts lors d'un challenge) ; page reçue sans
  identité = 404 rumble_video_not_found.
- Code mort supprimé : scrapeRumbleVideo d'index.mjs (191 l., 3e
  implémentation jamais appelée), rumbleLimiter 10/min jamais monté,
  import cheerio devenu inutile. Net -238 lignes.
- Tests : parseur générique (vues, durées, ids /shorts/), contrat 503
  sans réseau via le cooldown, durées absentes -> undefined.
- Docs : API_MCP_GUIDE §5.5 (routes, rate-limit réel, erreurs) + note
  README « Rumble peut être temporairement absent selon le réseau ».
This commit is contained in:
2026-10-02 08:36:48 -04:00
parent 4d1d44f4ee
commit e48d76be57
7 changed files with 496 additions and 734 deletions
-202
View File
@@ -12,7 +12,6 @@ import { execFile as execFileCb } from 'node:child_process';
import { promisify } from 'node:util';
import { fileURLToPath as serverFileURLToPath } from 'node:url';
import ffmpegPath from 'ffmpeg-static';
import * as cheerio from 'cheerio';
import axios from 'axios';
import rumbleRouter from './rumble.mjs';
import { providerRegistry, getProviderAdapter, validateProviders, SUGGESTION_CONTRACT_VERSION } from './providers/registry.mjs';
@@ -1140,15 +1139,6 @@ function formatsCacheSet(key, data) {
}
}
// Rate limiter for Rumble scraping to prevent being blocked
const rumbleLimiter = rateLimit({
windowMs: 60 * 1000, // 1 min
max: 10, // Limit to 10 requests per minute per IP
standardHeaders: true,
legacyHeaders: false,
message: { error: 'Too many requests to Rumble API. Please try again later.' }
});
function makeAccessToken(userId, sessionId) {
const payload = { sub: userId, sid: sessionId };
return jwt.sign(payload, JWT_SECRET, { expiresIn: `${ACCESS_TTL_MIN}m` });
@@ -1300,198 +1290,6 @@ function providerUrlFrom(provider, videoId, { instance, slug, sourceUrl }) {
}
}
async function scrapeRumbleVideo(videoId) {
const url = `https://rumble.com/${videoId}`;
try {
const response = await axios.get(url, {
headers: {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.5',
'Accept-Encoding': 'gzip, deflate, br',
'DNT': '1',
'Connection': 'keep-alive',
'Upgrade-Insecure-Requests': '1',
'Cache-Control': 'no-cache'
},
timeout: 15000,
maxRedirects: 5,
validateStatus: function (status) {
return status >= 200 && status < 400; // Accept redirects
}
});
const $ = cheerio.load(response.data);
const html = response.data;
// Extract basic video information
let title = '';
let thumbnail = '';
let uploaderName = '';
let uploaderAvatar = '';
let views = 0;
let duration = 0;
let uploadedDate = '';
let description = '';
// Try multiple selectors for title (Rumble's HTML structure can vary)
title = $('h1.video-title, .video-title h1, [data-video-title]').first().text().trim() ||
$('meta[property="og:title"]').attr('content') ||
$('title').text().trim() ||
$('h1').first().text().trim() || '';
// Clean up title (remove site name if present)
title = title.replace(/\s*\|\s*Rumble$/i, '').trim();
// Extract thumbnail with fallbacks
thumbnail = $('meta[property="og:image"], meta[name="twitter:image"]').attr('content') ||
$('meta[property="og:image:secure_url"]').attr('content') ||
$('.video-thumbnail img, .thumbnail img').attr('src') || '';
// Make thumbnail URL absolute if relative
if (thumbnail && !thumbnail.startsWith('http')) {
thumbnail = thumbnail.startsWith('//') ? 'https:' + thumbnail : 'https://rumble.com' + thumbnail;
}
// Extract uploader information with multiple selectors
uploaderName = $('.media-by--a, .channel-name, .uploader-name').first().text().trim() ||
$('meta[property="article:author"]').attr('content') ||
$('.author-name, .channel-link').first().text().trim() || '';
uploaderAvatar = $('.channel-avatar img, .uploader-avatar img').attr('src') ||
$('.author-avatar img').attr('src') || '';
// Make uploader avatar URL absolute
if (uploaderAvatar && !uploaderAvatar.startsWith('http')) {
uploaderAvatar = uploaderAvatar.startsWith('//') ? 'https:' + uploaderAvatar : 'https://rumble.com' + uploaderAvatar;
}
// Extract views with better parsing
const viewsText = $('.rumbles-views, .video-views, .views-count, .video-info .views').first().text().trim();
if (viewsText) {
const viewsMatch = viewsText.match(/([\d,]+(?:\.\d+)?)\s*(K|M|B)?/i);
if (viewsMatch) {
let num = parseFloat(viewsMatch[1].replace(/,/g, ''));
const multiplier = viewsMatch[2]?.toUpperCase();
if (multiplier === 'K') num *= 1000;
else if (multiplier === 'M') num *= 1000000;
else if (multiplier === 'B') num *= 1000000000;
views = Math.floor(num);
} else {
// Try direct number parsing
const directMatch = viewsText.match(/(\d+(?:,\d+)*)/);
if (directMatch) {
views = parseInt(directMatch[1].replace(/,/g, ''));
}
}
}
// Extract duration with improved parsing
const durationText = $('meta[property="video:duration"]').attr('content') ||
$('video').attr('duration') ||
$('.video-duration, .duration, .video-time').first().text().trim() ||
$('.time-duration').text().trim();
if (durationText) {
if (!isNaN(durationText)) {
duration = parseInt(durationText);
} else {
// Parse various duration formats
const timeMatch = durationText.match(/(\d+):(\d+)(?::(\d+))?/);
if (timeMatch) {
const hours = parseInt(timeMatch[3] || '0');
const minutes = parseInt(timeMatch[1]);
const seconds = parseInt(timeMatch[2]);
duration = hours * 3600 + minutes * 60 + seconds;
} else {
// Try HH:MM:SS format or MM:SS
const parts = durationText.split(':').map(p => parseInt(p.trim()) || 0);
if (parts.length === 3) {
duration = parts[0] * 3600 + parts[1] * 60 + parts[2];
} else if (parts.length === 2) {
duration = parts[0] * 60 + parts[1];
}
}
}
}
// Extract upload date
uploadedDate = $('meta[property="article:published_time"]').attr('content') ||
$('.upload-date, .published-date, .video-date').first().text().trim() || '';
// Try to parse relative dates
if (!uploadedDate || uploadedDate.includes('ago')) {
const relativeDate = $('.upload-date, .published-date').first().text().trim();
if (relativeDate && relativeDate.includes('ago')) {
// Convert relative date to ISO string (simple conversion)
const now = new Date();
if (relativeDate.includes('hour')) {
const hours = parseInt(relativeDate.match(/(\d+)/)?.[1] || '1');
now.setHours(now.getHours() - hours);
uploadedDate = now.toISOString();
} else if (relativeDate.includes('day')) {
const days = parseInt(relativeDate.match(/(\d+)/)?.[1] || '1');
now.setDate(now.getDate() - days);
uploadedDate = now.toISOString();
}
}
}
// Extract description
description = $('meta[property="og:description"]').attr('content') ||
$('.video-description, .description, .video-summary').first().text().trim() || '';
// Extract video ID from various sources
let extractedVideoId = videoId;
const videoIdMatch = html.match(/"video_id"\s*:\s*"([^"]+)"/) ||
html.match(/video[_-]id["\s:]+([^"\s]+)/) ||
html.match(/embed\/([^/?]+)/);
if (videoIdMatch && videoIdMatch[1]) {
extractedVideoId = videoIdMatch[1];
}
// Validate extracted data
const isValidVideo = title || thumbnail || uploaderName;
return {
videoId: extractedVideoId,
title: title || 'Untitled Video',
thumbnail,
uploaderName: uploaderName || 'Unknown Uploader',
uploaderAvatar: uploaderAvatar || thumbnail,
views: Math.max(0, views),
duration: Math.max(0, duration),
uploadedDate: uploadedDate || new Date().toISOString(),
description,
url,
type: 'video',
scraped: true,
confidence: isValidVideo ? 'high' : 'low'
};
} catch (error) {
console.error('Erreur scraping Rumble:', error.message);
// Return minimal data for fallback with error info
return {
videoId,
title: 'Video unavailable',
thumbnail: '',
uploaderName: 'Unknown',
uploaderAvatar: '',
views: 0,
duration: 0,
uploadedDate: '',
description: '',
url,
type: 'video',
error: error.message,
scraped: false,
confidence: 'none'
};
}
}
function guessContentTypeByExt(ext) {
const e = String(ext || '').toLowerCase();
if (e === 'mp4' || e === 'm4v') return 'video/mp4';