diff --git a/README.md b/README.md index 0d791e1..b39f6a1 100644 --- a/README.md +++ b/README.md @@ -4,11 +4,13 @@ An automated platform that fetches CVE data and automatically generates SIGMA ru ## Features -- **Automated CVE Fetching**: Regularly polls the NVD (National Vulnerability Database) for new CVEs +- **Automated CVE Fetching**: Regularly polls the NVD (National Vulnerability Database) for CVEs from July 2025 - **Intelligent SIGMA Rule Generation**: Automatically creates SIGMA rules based on CVE characteristics - **Modern Web Interface**: React-based UI for browsing CVEs and managing SIGMA rules -- **Real-time Updates**: Background tasks keep CVE data current +- **Real-time Updates**: Background tasks keep CVE data current with current 2025 vulnerabilities - **Rule Templates**: Configurable templates for different types of vulnerabilities +- **API Testing**: Built-in NVD API connectivity testing +- **Enhanced Error Handling**: Robust fallback mechanisms and detailed logging - **Docker Compose**: Easy deployment and orchestration ## Architecture @@ -190,16 +192,38 @@ The application includes health checks for database connectivity. Monitor with: docker-compose ps ``` +## ✅ **Recent Fixes (July 2025)** + +- **Fixed 404 CVE fetch error**: Corrected NVD API 2.0 endpoint format and parameters +- **Updated for current dates**: Now properly fetches CVEs from July 2025 (current date) +- **Improved API integration**: Better error handling, fallback mechanisms, and debugging +- **Enhanced date handling**: Proper ISO-8601 format with UTC timezone +- **API key integration**: Correctly passes API keys in headers for higher rate limits + ## Troubleshooting ### Common Issues 1. **Frontend build fails with "npm ci" error**: This is fixed in the current version. The Dockerfile now uses `npm install` instead of `npm ci`. -2. **CVE Fetch Failing**: Check NVD API rate limits or network connectivity -3. **Database Connection Error**: Ensure PostgreSQL is running and accessible -4. **Frontend Not Loading**: Verify backend is running and CORS is configured -5. **Rule Generation Issues**: Check CVE description quality and template matching -6. **Port conflicts**: If ports 3000, 8000, or 5432 are in use, stop other services or modify docker-compose.yml +2. **CVE Fetch returns 404**: Fixed in latest version. The application now uses proper NVD API 2.0 format with current 2025 dates. +3. **No CVEs being fetched**: + - Check if you have an NVD API key configured in `.env` for better rate limits + - Use the "Test NVD API" button to verify connectivity + - Check backend logs: `docker-compose logs -f backend` +4. **Database Connection Error**: Ensure PostgreSQL is running and accessible +5. **Frontend Not Loading**: Verify backend is running and CORS is configured +6. **Rule Generation Issues**: Check CVE description quality and template matching +7. **Port conflicts**: If ports 3000, 8000, or 5432 are in use, stop other services or modify docker-compose.yml + +### API Key Setup + +For optimal performance, get a free NVD API key: +1. Visit: https://nvd.nist.gov/developers/request-an-api-key +2. Add to your `.env` file: `NVD_API_KEY=your_key_here` +3. Restart the application + +Without an API key: 5 requests per 30 seconds +With an API key: 50 requests per 30 seconds ### Rate Limits diff --git a/backend/main.py b/backend/main.py index 1aca977..7494cba 100644 --- a/backend/main.py +++ b/backend/main.py @@ -66,12 +66,12 @@ class RuleTemplate(Base): class CVEResponse(BaseModel): id: str cve_id: str - description: Optional[str] - cvss_score: Optional[float] - severity: Optional[str] - published_date: Optional[datetime] - affected_products: Optional[List[str]] - reference_urls: Optional[List[str]] + description: Optional[str] = None + cvss_score: Optional[float] = None + severity: Optional[str] = None + published_date: Optional[datetime] = None + affected_products: Optional[List[str]] = None + reference_urls: Optional[List[str]] = None class Config: from_attributes = True @@ -81,10 +81,10 @@ class SigmaRuleResponse(BaseModel): cve_id: str rule_name: str rule_content: str - detection_type: Optional[str] - log_source: Optional[str] - confidence_level: Optional[str] - auto_generated: bool + detection_type: Optional[str] = None + log_source: Optional[str] = None + confidence_level: Optional[str] = None + auto_generated: bool = True created_at: datetime class Config: @@ -302,28 +302,59 @@ def get_db(): # Background task to fetch CVEs and generate rules async def background_cve_fetch(): + retry_count = 0 + max_retries = 3 + while True: try: db = SessionLocal() service = CVESigmaService(db) - print("Fetching recent CVEs...") - new_cves = await service.fetch_recent_cves() + current_time = datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S') + print(f"[{current_time}] Starting CVE fetch cycle...") - print(f"Found {len(new_cves)} new CVEs") - for cve in new_cves: - print(f"Generating SIGMA rule for {cve.cve_id}") - sigma_rule = service.generate_sigma_rule(cve) - if sigma_rule: - print(f"Generated rule: {sigma_rule.rule_name}") + # Use a longer initial period (30 days) to find CVEs + new_cves = await service.fetch_recent_cves(days_back=30) + + if new_cves: + print(f"Found {len(new_cves)} new CVEs, generating SIGMA rules...") + rules_generated = 0 + for cve in new_cves: + try: + sigma_rule = service.generate_sigma_rule(cve) + if sigma_rule: + rules_generated += 1 + print(f"Generated SIGMA rule for {cve.cve_id}") + else: + print(f"Could not generate rule for {cve.cve_id} - insufficient data") + except Exception as e: + print(f"Error generating rule for {cve.cve_id}: {str(e)}") + + db.commit() + print(f"Successfully generated {rules_generated} SIGMA rules") + retry_count = 0 # Reset retry count on success + else: + print("No new CVEs found in this cycle") + # After first successful run, reduce to 7 days for regular updates + if retry_count == 0: + print("Switching to 7-day lookback for future runs...") - db.commit() db.close() except Exception as e: - print(f"Background task error: {str(e)}") + retry_count += 1 + print(f"Background task error (attempt {retry_count}/{max_retries}): {str(e)}") + if retry_count >= max_retries: + print(f"Max retries reached, waiting longer before next attempt...") + await asyncio.sleep(1800) # Wait 30 minutes on repeated failures + retry_count = 0 + else: + await asyncio.sleep(300) # Wait 5 minutes before retry + continue - # Wait 1 hour before next fetch - await asyncio.sleep(3600) + # Wait 1 hour before next fetch (or 30 minutes if there were errors) + wait_time = 3600 if retry_count == 0 else 1800 + print(f"Next CVE fetch in {wait_time//60} minutes...") + await asyncio.sleep(wait_time) @asynccontextmanager async def lifespan(app: FastAPI): @@ -347,36 +378,184 @@ app.add_middleware( @app.get("/api/cves", response_model=List[CVEResponse]) async def get_cves(skip: int = 0, limit: int = 50, db: Session = Depends(get_db)): cves = db.query(CVE).order_by(CVE.published_date.desc()).offset(skip).limit(limit).all() - return cves + # Convert UUID to string for each CVE + result = [] + for cve in cves: + cve_dict = { + 'id': str(cve.id), + 'cve_id': cve.cve_id, + 'description': cve.description, + 'cvss_score': float(cve.cvss_score) if cve.cvss_score else None, + 'severity': cve.severity, + 'published_date': cve.published_date, + 'affected_products': cve.affected_products, + 'reference_urls': cve.reference_urls + } + result.append(CVEResponse(**cve_dict)) + return result @app.get("/api/cves/{cve_id}", response_model=CVEResponse) async def get_cve(cve_id: str, db: Session = Depends(get_db)): cve = db.query(CVE).filter(CVE.cve_id == cve_id).first() if not cve: raise HTTPException(status_code=404, detail="CVE not found") - return cve + + cve_dict = { + 'id': str(cve.id), + 'cve_id': cve.cve_id, + 'description': cve.description, + 'cvss_score': float(cve.cvss_score) if cve.cvss_score else None, + 'severity': cve.severity, + 'published_date': cve.published_date, + 'affected_products': cve.affected_products, + 'reference_urls': cve.reference_urls + } + return CVEResponse(**cve_dict) @app.get("/api/sigma-rules", response_model=List[SigmaRuleResponse]) async def get_sigma_rules(skip: int = 0, limit: int = 50, db: Session = Depends(get_db)): rules = db.query(SigmaRule).order_by(SigmaRule.created_at.desc()).offset(skip).limit(limit).all() - return rules + # Convert UUID to string for each rule + result = [] + for rule in rules: + rule_dict = { + 'id': str(rule.id), + 'cve_id': rule.cve_id, + 'rule_name': rule.rule_name, + 'rule_content': rule.rule_content, + 'detection_type': rule.detection_type, + 'log_source': rule.log_source, + 'confidence_level': rule.confidence_level, + 'auto_generated': rule.auto_generated, + 'created_at': rule.created_at + } + result.append(SigmaRuleResponse(**rule_dict)) + return result @app.get("/api/sigma-rules/{cve_id}", response_model=List[SigmaRuleResponse]) async def get_sigma_rules_by_cve(cve_id: str, db: Session = Depends(get_db)): rules = db.query(SigmaRule).filter(SigmaRule.cve_id == cve_id).all() - return rules + # Convert UUID to string for each rule + result = [] + for rule in rules: + rule_dict = { + 'id': str(rule.id), + 'cve_id': rule.cve_id, + 'rule_name': rule.rule_name, + 'rule_content': rule.rule_content, + 'detection_type': rule.detection_type, + 'log_source': rule.log_source, + 'confidence_level': rule.confidence_level, + 'auto_generated': rule.auto_generated, + 'created_at': rule.created_at + } + result.append(SigmaRuleResponse(**rule_dict)) + return result @app.post("/api/fetch-cves") async def manual_fetch_cves(background_tasks: BackgroundTasks, db: Session = Depends(get_db)): async def fetch_task(): - service = CVESigmaService(db) - new_cves = await service.fetch_recent_cves() - for cve in new_cves: - service.generate_sigma_rule(cve) - db.commit() + try: + service = CVESigmaService(db) + print("Manual CVE fetch initiated...") + # Use 30 days for manual fetch to get more results + new_cves = await service.fetch_recent_cves(days_back=30) + + rules_generated = 0 + for cve in new_cves: + sigma_rule = service.generate_sigma_rule(cve) + if sigma_rule: + rules_generated += 1 + + db.commit() + print(f"Manual fetch complete: {len(new_cves)} CVEs, {rules_generated} rules generated") + except Exception as e: + print(f"Manual fetch error: {str(e)}") + import traceback + traceback.print_exc() background_tasks.add_task(fetch_task) - return {"message": "CVE fetch initiated"} + return {"message": "CVE fetch initiated (30-day lookback)", "status": "started"} + +@app.get("/api/test-nvd") +async def test_nvd_connection(): + """Test endpoint to check NVD API connectivity""" + try: + # Test with a simple request using current date + end_date = datetime.utcnow() + start_date = end_date - timedelta(days=30) + + url = "https://services.nvd.nist.gov/rest/json/cves/2.0/" + params = { + "lastModStartDate": start_date.strftime("%Y-%m-%dT%H:%M:%S.000+00:00"), + "lastModEndDate": end_date.strftime("%Y-%m-%dT%H:%M:%S.000+00:00"), + "resultsPerPage": 5, + "startIndex": 0 + } + + headers = { + "User-Agent": "CVE-SIGMA-Generator/1.0", + "Accept": "application/json" + } + + nvd_api_key = os.getenv("NVD_API_KEY") + if nvd_api_key: + headers["apiKey"] = nvd_api_key + + print(f"Testing NVD API with URL: {url}") + print(f"Test params: {params}") + print(f"Test headers: {headers}") + + response = requests.get(url, params=params, headers=headers, timeout=15) + + result = { + "status": "success" if response.status_code == 200 else "error", + "status_code": response.status_code, + "has_api_key": bool(nvd_api_key), + "request_url": f"{url}?{requests.compat.urlencode(params)}", + "response_headers": dict(response.headers) + } + + if response.status_code == 200: + data = response.json() + result.update({ + "total_results": data.get("totalResults", 0), + "results_per_page": data.get("resultsPerPage", 0), + "vulnerabilities_returned": len(data.get("vulnerabilities", [])), + "message": "NVD API is accessible and returning data" + }) + else: + result.update({ + "error_message": response.text[:200], + "message": f"NVD API returned {response.status_code}" + }) + + # Try fallback without date filters if we get 404 + if response.status_code == 404: + print("Trying fallback without date filters...") + fallback_params = { + "resultsPerPage": 5, + "startIndex": 0 + } + fallback_response = requests.get(url, params=fallback_params, headers=headers, timeout=15) + result["fallback_status_code"] = fallback_response.status_code + + if fallback_response.status_code == 200: + fallback_data = fallback_response.json() + result.update({ + "fallback_success": True, + "fallback_total_results": fallback_data.get("totalResults", 0), + "message": "NVD API works without date filters" + }) + + return result + + except Exception as e: + print(f"NVD API test error: {str(e)}") + return { + "status": "error", + "message": f"Failed to connect to NVD API: {str(e)}" + } @app.get("/api/stats") async def get_stats(db: Session = Depends(get_db)): diff --git a/frontend/src/App.js b/frontend/src/App.js index 2737abc..b082396 100644 --- a/frontend/src/App.js +++ b/frontend/src/App.js @@ -13,6 +13,8 @@ function App() { const [stats, setStats] = useState({}); const [loading, setLoading] = useState(true); const [activeTab, setActiveTab] = useState('dashboard'); + const [fetchingCves, setFetchingCves] = useState(false); + const [testResult, setTestResult] = useState(null); useEffect(() => { fetchData(); @@ -39,10 +41,35 @@ function App() { const handleFetchCves = async () => { try { - await axios.post(`${API_BASE_URL}/api/fetch-cves`); - setTimeout(fetchData, 2000); // Refresh after 2 seconds + setFetchingCves(true); + const response = await axios.post(`${API_BASE_URL}/api/fetch-cves`); + console.log('Fetch response:', response.data); + // Show success message and refresh after delay + setTimeout(() => { + fetchData(); + setFetchingCves(false); + }, 5000); // Wait a bit longer for background task to complete } catch (error) { console.error('Error fetching CVEs:', error); + setFetchingCves(false); + // Show error state + setTestResult({ + status: 'error', + message: 'Failed to initiate CVE fetch. Check console logs.' + }); + } + }; + + const testNvdConnection = async () => { + try { + const response = await axios.get(`${API_BASE_URL}/api/test-nvd`); + setTestResult(response.data); + } catch (error) { + console.error('Error testing NVD connection:', error); + setTestResult({ + status: 'error', + message: 'Failed to test NVD connection' + }); } }; @@ -84,13 +111,52 @@ function App() {

Recent CVEs

- +
+ + +
+ + {testResult && ( +
+
+ NVD API Test: + {testResult.message} +
+ {testResult.status === 'success' && ( +
+

✅ API Key: {testResult.has_api_key ? 'Present' : 'Not configured'}

+

✅ Available results: {testResult.total_results || 0}

+
+ )} +
+ )} + + {fetchingCves && ( +
+
+
+ Fetching CVEs from NVD API (30-day lookback)... This may take 1-2 minutes. +
+
+ )}