116 lines
3.7 KiB
JavaScript
116 lines
3.7 KiB
JavaScript
require('dotenv').config({ path: require('path').join(__dirname, '..', '.env') });
|
|
const Parser = require('rss-parser');
|
|
const fs = require('fs');
|
|
const path = require('path');
|
|
const { db } = require('../app/db');
|
|
const { v4: uuidv4 } = require('uuid');
|
|
|
|
const parser = new Parser({
|
|
customFields: {
|
|
item: [
|
|
['content:encoded', 'contentEncoded'],
|
|
['dc:creator', 'creator'],
|
|
['media:content', 'media']
|
|
]
|
|
}
|
|
});
|
|
|
|
function normalizeText(text) {
|
|
if (!text) return '';
|
|
// Supprimer HTML basique
|
|
return text.replace(/<[^>]*>/g, ' ').replace(/\s+/g, ' ').trim().substring(0, 500);
|
|
}
|
|
|
|
function getCategoryFromSource(hint) {
|
|
if (hint) return [hint];
|
|
return ['Tech'];
|
|
}
|
|
|
|
async function collect() {
|
|
console.log('[COLLECT] Démarrage de la collecte...');
|
|
|
|
const sourcesPath = path.join(__dirname, '..', 'config', 'sources.json');
|
|
let sources = [];
|
|
try {
|
|
sources = JSON.parse(fs.readFileSync(sourcesPath, 'utf8'));
|
|
} catch(e) {
|
|
console.error('[COLLECT] Erreur lecture sources:', e.message);
|
|
return;
|
|
}
|
|
|
|
const autoPublish = process.env.AUTO_PUBLISH === 'true';
|
|
let totalAdded = 0;
|
|
let totalSkipped = 0;
|
|
|
|
for (const source of sources) {
|
|
if (source.enabled === false) continue;
|
|
|
|
try {
|
|
console.log(`[COLLECT] Récupération depuis ${source.name}...`);
|
|
const feed = await parser.parseURL(source.url);
|
|
|
|
for (const item of feed.items || []) {
|
|
const url = item.link || item.guid;
|
|
if (!url) continue;
|
|
|
|
// Vérifier doublon par URL
|
|
const existing = db.prepare('SELECT id FROM articles WHERE url = ?').get(url);
|
|
if (existing) {
|
|
totalSkipped++;
|
|
continue;
|
|
}
|
|
|
|
// Vérifier doublon par titre très proche (simple)
|
|
const title = item.title || '';
|
|
if (title) {
|
|
const similar = db.prepare('SELECT id FROM articles WHERE LOWER(title) = LOWER(?)').get(title);
|
|
if (similar) {
|
|
totalSkipped++;
|
|
continue;
|
|
}
|
|
}
|
|
|
|
const id = uuidv4();
|
|
const publishedAt = item.pubDate || item.isoDate || new Date().toISOString();
|
|
const summary = normalizeText(item.contentSnippet || item.contentEncoded || item.description || '');
|
|
const categories = JSON.stringify(getCategoryFromSource(source.category_hint));
|
|
const status = autoPublish ? 'published' : 'pending';
|
|
const lang = source.language || 'fr';
|
|
|
|
try {
|
|
db.prepare(`INSERT INTO articles (id, title, url, source, source_language, language, published_at, summary, categories, status, collected_by)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'rss')`)
|
|
.run(id, title, url, source.name, lang, lang, publishedAt, summary, categories, status);
|
|
|
|
totalAdded++;
|
|
} catch(e) {
|
|
console.error(`[COLLECT] Erreur insertion:`, e.message);
|
|
}
|
|
}
|
|
|
|
// Mettre à jour last_collected_at si table sources existe
|
|
try {
|
|
db.prepare(`UPDATE sources SET last_collected_at = datetime('now') WHERE id = ?`).run(source.id);
|
|
} catch(e) {}
|
|
|
|
} catch(e) {
|
|
console.error(`[COLLECT] Erreur pour ${source.name}:`, e.message);
|
|
}
|
|
}
|
|
|
|
console.log(`[COLLECT] Terminé. Ajoutés: ${totalAdded}, Ignorés (doublons): ${totalSkipped}`);
|
|
|
|
try {
|
|
db.prepare(`INSERT INTO activity_log (action, details) VALUES ('collect', ?)`).run(`Collecte automatique: ${totalAdded} ajoutés, ${totalSkipped} ignorés`);
|
|
} catch(e) {}
|
|
}
|
|
|
|
if (require.main === module) {
|
|
collect().then(() => process.exit(0)).catch(err => {
|
|
console.error(err);
|
|
process.exit(1);
|
|
});
|
|
}
|
|
|
|
module.exports = { collect };
|