feat: add base collector infrastructure for data collection jobs
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
parent
61e2748e9d
commit
3abbdfa5b6
@ -0,0 +1,3 @@
|
||||
from app.services.collectors import BaseCollector
|
||||
|
||||
__all__ = ["BaseCollector"]
|
||||
3
backend/app/services/collectors/__init__.py
Normal file
3
backend/app/services/collectors/__init__.py
Normal file
@ -0,0 +1,3 @@
|
||||
from app.services.collectors.base import BaseCollector
|
||||
|
||||
__all__ = ["BaseCollector"]
|
||||
65
backend/app/services/collectors/base.py
Normal file
65
backend/app/services/collectors/base.py
Normal file
@ -0,0 +1,65 @@
|
||||
"""
|
||||
Base collector class for data collection jobs.
|
||||
"""
|
||||
from abc import ABC, abstractmethod
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models.stock import JobLog
|
||||
|
||||
|
||||
class BaseCollector(ABC):
|
||||
"""Base class for all data collectors."""
|
||||
|
||||
def __init__(self, db: Session):
|
||||
self.db = db
|
||||
self.job_name = self.__class__.__name__
|
||||
self.job_log: Optional[JobLog] = None
|
||||
|
||||
def start_job(self) -> JobLog:
|
||||
"""Create a job log entry when starting."""
|
||||
self.job_log = JobLog(
|
||||
job_name=self.job_name,
|
||||
status="running",
|
||||
started_at=datetime.utcnow(),
|
||||
)
|
||||
self.db.add(self.job_log)
|
||||
self.db.commit()
|
||||
return self.job_log
|
||||
|
||||
def complete_job(self, records_count: int):
|
||||
"""Mark job as completed."""
|
||||
if self.job_log:
|
||||
self.job_log.status = "success"
|
||||
self.job_log.finished_at = datetime.utcnow()
|
||||
self.job_log.records_count = records_count
|
||||
self.db.commit()
|
||||
|
||||
def fail_job(self, error_msg: str):
|
||||
"""Mark job as failed."""
|
||||
if self.job_log:
|
||||
self.job_log.status = "failed"
|
||||
self.job_log.finished_at = datetime.utcnow()
|
||||
self.job_log.error_msg = error_msg
|
||||
self.db.commit()
|
||||
|
||||
@abstractmethod
|
||||
def collect(self) -> int:
|
||||
"""
|
||||
Perform the data collection.
|
||||
Returns the number of records collected.
|
||||
"""
|
||||
pass
|
||||
|
||||
def run(self) -> JobLog:
|
||||
"""Execute the collection job with logging."""
|
||||
self.start_job()
|
||||
try:
|
||||
records = self.collect()
|
||||
self.complete_job(records)
|
||||
except Exception as e:
|
||||
self.fail_job(str(e))
|
||||
raise
|
||||
return self.job_log
|
||||
Loading…
x
Reference in New Issue
Block a user