Début du projet

collector/index.js

@@ -0,0 +1,115 @@
+require('dotenv').config({ path: require('path').join(__dirname, '..', '.env') });
+const Parser = require('rss-parser');
+const fs = require('fs');
+const path = require('path');
+const { db } = require('../app/db');
+const { v4: uuidv4 } = require('uuid');
+
+const parser = new Parser({
+  customFields: {
+    item: [
+      ['content:encoded', 'contentEncoded'],
+      ['dc:creator', 'creator'],
+      ['media:content', 'media']
+    ]
+  }
+});
+
+function normalizeText(text) {
+  if (!text) return '';
+  // Supprimer HTML basique
+  return text.replace(/<[^>]*>/g, ' ').replace(/\s+/g, ' ').trim().substring(0, 500);
+}
+
+function getCategoryFromSource(hint) {
+  if (hint) return [hint];
+  return ['Tech'];
+}
+
+async function collect() {
+  console.log('[COLLECT] Démarrage de la collecte...');
+  
+  const sourcesPath = path.join(__dirname, '..', 'config', 'sources.json');
+  let sources = [];
+  try {
+    sources = JSON.parse(fs.readFileSync(sourcesPath, 'utf8'));
+  } catch(e) {
+    console.error('[COLLECT] Erreur lecture sources:', e.message);
+    return;
+  }
+  
+  const autoPublish = process.env.AUTO_PUBLISH === 'true';
+  let totalAdded = 0;
+  let totalSkipped = 0;
+  
+  for (const source of sources) {
+    if (source.enabled === false) continue;
+    
+    try {
+      console.log(`[COLLECT] Récupération depuis ${source.name}...`);
+      const feed = await parser.parseURL(source.url);
+      
+      for (const item of feed.items || []) {
+        const url = item.link || item.guid;
+        if (!url) continue;
+        
+        // Vérifier doublon par URL
+        const existing = db.prepare('SELECT id FROM articles WHERE url = ?').get(url);
+        if (existing) {
+          totalSkipped++;
+          continue;
+        }
+        
+        // Vérifier doublon par titre très proche (simple)
+        const title = item.title || '';
+        if (title) {
+          const similar = db.prepare('SELECT id FROM articles WHERE LOWER(title) = LOWER(?)').get(title);
+          if (similar) {
+            totalSkipped++;
+            continue;
+          }
+        }
+        
+        const id = uuidv4();
+        const publishedAt = item.pubDate || item.isoDate || new Date().toISOString();
+        const summary = normalizeText(item.contentSnippet || item.contentEncoded || item.description || '');
+        const categories = JSON.stringify(getCategoryFromSource(source.category_hint));
+        const status = autoPublish ? 'published' : 'pending';
+        const lang = source.language || 'fr';
+        
+        try {
+          db.prepare(`INSERT INTO articles (id, title, url, source, source_language, language, published_at, summary, categories, status, collected_by)
+                      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'rss')`)
+            .run(id, title, url, source.name, lang, lang, publishedAt, summary, categories, status);
+          
+          totalAdded++;
+        } catch(e) {
+          console.error(`[COLLECT] Erreur insertion:`, e.message);
+        }
+      }
+      
+      // Mettre à jour last_collected_at si table sources existe
+      try {
+        db.prepare(`UPDATE sources SET last_collected_at = datetime('now') WHERE id = ?`).run(source.id);
+      } catch(e) {}
+      
+    } catch(e) {
+      console.error(`[COLLECT] Erreur pour ${source.name}:`, e.message);
+    }
+  }
+  
+  console.log(`[COLLECT] Terminé. Ajoutés: ${totalAdded}, Ignorés (doublons): ${totalSkipped}`);
+  
+  try {
+    db.prepare(`INSERT INTO activity_log (action, details) VALUES ('collect', ?)`).run(`Collecte automatique: ${totalAdded} ajoutés, ${totalSkipped} ignorés`);
+  } catch(e) {}
+}
+
+if (require.main === module) {
+  collect().then(() => process.exit(0)).catch(err => {
+    console.error(err);
+    process.exit(1);
+  });
+}
+
+module.exports = { collect };