fix: bulk fallback price collection

This commit is contained in:
머니페니 2026-05-14 23:40:55 +09:00
parent 5a452e4714
commit 0bcf5bbf23

View File

@ -147,58 +147,61 @@ class PriceCollector(BaseCollector):
return total_records return total_records
def _collect_pykrx(self) -> int: def _collect_pykrx(self) -> int:
"""Collect stock prices via pykrx scraping (ticker-based loop).""" """Collect stock prices via pykrx scraping (date/market bulk fetch)."""
from pykrx import stock as pykrx_stock from pykrx import stock as pykrx_stock
tickers = self.db.query(Stock.ticker).all() tickers = self.db.query(Stock.ticker, Stock.market).all()
ticker_list = [t[0] for t in tickers] ticker_market = {ticker: market for ticker, market in tickers}
if not ticker_list: if not ticker_market:
logger.warning("No stocks found in database. Run StockCollector first.") logger.warning("No stocks found in database. Run StockCollector first.")
return 0 return 0
total_records = 0 total_records = 0
logger.info(f"Collecting prices for {len(ticker_list)} stocks from {self.start_date} to {self.end_date}") logger.info(
"Collecting prices for %d stocks from %s to %s",
for ticker in ticker_list: len(ticker_market), self.start_date, self.end_date,
try:
df = pykrx_stock.get_market_ohlcv(
self.start_date, self.end_date, ticker
) )
if df.empty:
continue
df = df.reset_index() start = datetime.strptime(self.start_date, "%Y%m%d")
df.columns = ["date", "open", "high", "low", "close", "volume", end = datetime.strptime(self.end_date, "%Y%m%d")
"value"] current = start
expected_cols = 7 def get_value(row, *names):
if len(df.columns) < expected_cols: for name in names:
logger.warning(f"Unexpected column count for {ticker}: {len(df.columns)}") if name in row:
return row[name]
return None
while current <= end:
date_str = current.strftime("%Y%m%d")
date_value = current.date()
for market in ("KOSPI", "KOSDAQ"):
try:
df = pykrx_stock.get_market_ohlcv(date_str, market=market)
if df is None or df.empty:
continue continue
records = [] records = []
for _, row in df.iterrows(): for ticker, row in df.iterrows():
open_val = self._safe_float(row["open"]) ticker = str(ticker).zfill(6)
high_val = self._safe_float(row["high"]) if ticker_market.get(ticker) != market:
low_val = self._safe_float(row["low"]) continue
close_val = self._safe_float(row["close"])
volume_val = self._safe_int(row["volume"]) close_val = self._safe_float(get_value(row, "종가", "close"))
if close_val is None or close_val == 0:
if close_val is None:
logger.debug(f"Skipping record for {ticker}: missing close price")
continue continue
date_value = row["date"].date() if hasattr(row["date"], "date") else row["date"]
records.append({ records.append({
"ticker": ticker, "ticker": ticker,
"date": date_value, "date": date_value,
"open": open_val, "open": self._safe_float(get_value(row, "시가", "open")),
"high": high_val, "high": self._safe_float(get_value(row, "고가", "high")),
"low": low_val, "low": self._safe_float(get_value(row, "저가", "low")),
"close": close_val, "close": close_val,
"volume": volume_val, "volume": self._safe_int(get_value(row, "거래량", "volume")),
"trading_value": self._safe_int(row["value"]), "trading_value": self._safe_int(get_value(row, "거래대금", "value", "trading_value")),
}) })
if records: if records:
@ -221,15 +224,18 @@ class PriceCollector(BaseCollector):
except JSONDecodeError as e: except JSONDecodeError as e:
self.db.rollback() self.db.rollback()
logger.warning( logger.warning(
f"Price fetch for {ticker}: JSON decode error ({e}). " "Price fetch for %s %s: JSON decode error (%s). "
"KRX may require login — set KRX_ID/KRX_PW env vars." "KRX may require login — set KRX_ID/KRX_PW env vars.",
market, date_str, e,
) )
continue continue
except Exception as e: except Exception as e:
self.db.rollback() self.db.rollback()
logger.warning(f"Failed to fetch prices for {ticker}: {e}") logger.warning("Failed to fetch prices for %s %s: %s", market, date_str, e)
continue continue
current += timedelta(days=1)
return total_records return total_records
def collect(self) -> int: def collect(self) -> int: