openstock/src/providers/kind/universe.rs

use crate::core::stock::{
    infer_stock_kind, normalize_kind_market, validate_symbol, Stock, StockId, StockKind,
    StockMarket, StockSource,
};
use encoding_rs::EUC_KR;
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
use std::fs;
use std::path::{Path, PathBuf};
use std::time::{SystemTime, UNIX_EPOCH};

const KIND_UNIVERSE_URL: &str =
    "https://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13";
const CACHE_DIR: &str = ".openstock/universe/kind";
const LATEST_FILE: &str = "latest.json";
const META_FILE: &str = "meta.json";
const SNAPSHOT_RETENTION_FILES: usize = 7;
const SNAPSHOT_RETENTION_BYTES: u64 = 25 * 1024 * 1024;

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct UniverseMeta {
    pub source: String,
    pub source_url: String,
    pub date: String,
    pub refreshed_at: String,
    pub stock_count: usize,
    pub counts_by_market: BTreeMap<String, usize>,
}

#[derive(Debug, Clone)]
pub struct UniverseSnapshot {
    pub stocks: Vec<Stock>,
    pub meta: UniverseMeta,
    pub refreshed: bool,
}

pub fn load_or_refresh(force: bool) -> Result<UniverseSnapshot, String> {
    if !force {
        if let Some(snapshot) = load_today_cache()? {
            return Ok(snapshot);
        }
    }

    match refresh() {
        Ok(snapshot) => Ok(snapshot),
        Err(err) => {
            if let Some(snapshot) = load_any_cache()? {
                return Ok(UniverseSnapshot {
                    refreshed: false,
                    ..snapshot
                });
            }
            Err(err)
        }
    }
}

pub fn refresh() -> Result<UniverseSnapshot, String> {
    let body = download()?;
    let html = decode_euc_kr(&body);
    let mut stocks = parse_kind_html(&html)?;
    stocks.sort_by(|a, b| {
        a.market
            .cmp(&b.market)
            .then_with(|| a.kind.cmp(&b.kind))
            .then_with(|| a.id.symbol.cmp(&b.id.symbol))
    });

    let today = current_utc_date_string();
    for stock in &mut stocks {
        stock.updated_at = Some(today.clone());
    }

    let meta = UniverseMeta {
        source: "KIND".to_string(),
        source_url: KIND_UNIVERSE_URL.to_string(),
        date: today.clone(),
        refreshed_at: current_utc_datetime_string(),
        stock_count: stocks.len(),
        counts_by_market: counts_by_market(&stocks),
    };

    write_cache(&stocks, &meta)?;

    Ok(UniverseSnapshot {
        stocks,
        meta,
        refreshed: true,
    })
}

pub fn load_cached() -> Result<Option<UniverseSnapshot>, String> {
    load_any_cache()
}

pub fn prune_cache(dry_run: bool) -> Result<crate::core::cache::CachePruneReport, String> {
    crate::core::cache::prune_json_snapshots(
        "universe/kind",
        &cache_dir(),
        &[LATEST_FILE, META_FILE],
        SNAPSHOT_RETENTION_FILES,
        SNAPSHOT_RETENTION_BYTES,
        dry_run,
    )
}

pub fn chunk_stocks(stocks: &[Stock], size: usize) -> Vec<serde_json::Value> {
    let size = size.max(1);
    let mut groups = BTreeMap::<(StockMarket, StockKind), Vec<&Stock>>::new();
    for stock in stocks {
        groups
            .entry((stock.market, stock.kind))
            .or_default()
            .push(stock);
    }

    let mut chunks = Vec::new();
    for ((market, kind), group) in groups {
        for (index, chunk) in group.chunks(size).enumerate() {
            let first = chunk.first().copied();
            let last = chunk.last().copied();
            chunks.push(serde_json::json!({
                "chunk_id": format!("{}:{}:{:04}", market.as_str(), kind.as_str(), index + 1),
                "index": index + 1,
                "size": size,
                "count": chunk.len(),
                "start_symbol": first.map(|stock| stock.id.symbol.as_str()),
                "end_symbol": last.map(|stock| stock.id.symbol.as_str()),
                "market": market.as_str(),
                "kind": kind.as_str(),
            }));
        }
    }

    chunks
}

fn download() -> Result<Vec<u8>, String> {
    let response = crate::core::http::agent()
        .get(KIND_UNIVERSE_URL)
        .header("User-Agent", "openstock/0.1")
        .call()
        .map_err(|err| format!("[KIND] 상장법인목록 다운로드 실패: {}", err))?;
    let status = response.status();
    let body = response
        .into_body()
        .read_to_vec()
        .map_err(|err| format!("[KIND] 상장법인목록 응답 읽기 실패: {}", err))?;

    if !status.is_success() {
        return Err(format!(
            "[KIND] 상장법인목록 다운로드 오류 ({}): {} bytes",
            status,
            body.len()
        ));
    }

    Ok(body)
}

fn decode_euc_kr(bytes: &[u8]) -> String {
    let (decoded, _, _) = EUC_KR.decode(bytes);
    decoded.into_owned()
}

fn parse_kind_html(html: &str) -> Result<Vec<Stock>, String> {
    let mut rows = Vec::new();
    let mut rest = html;

    while let Some(start) = find_ascii_case_insensitive(rest, "<tr") {
        let row_start = &rest[start..];
        let Some(open_end) = row_start.find('>') else {
            break;
        };
        let row_body_start = open_end + 1;
        let Some(close) = find_ascii_case_insensitive(&row_start[row_body_start..], "</tr>") else {
            break;
        };
        let row_body = &row_start[row_body_start..row_body_start + close];
        let cells = extract_cells(row_body);
        if cells.len() >= 10 && cells[0] != "회사명" {
            if let Some(stock) = row_to_stock(&cells) {
                rows.push(stock);
            }
        }
        rest = &row_start[row_body_start + close + "</tr>".len()..];
    }

    if rows.is_empty() {
        return Err("[KIND] 상장법인목록에서 종목을 찾지 못했습니다.".to_string());
    }

    Ok(rows)
}

fn row_to_stock(cells: &[String]) -> Option<Stock> {
    let name = cells.first()?.trim();
    let market = cells.get(1)?.trim();
    let symbol = cells.get(2)?.trim();
    let industry = empty_to_none(cells.get(3).map(String::as_str).unwrap_or(""));
    let main_product = empty_to_none(cells.get(4).map(String::as_str).unwrap_or(""));

    if name.is_empty() || validate_symbol(symbol).is_err() {
        return None;
    }

    Some(Stock {
        id: StockId {
            symbol: symbol.to_string(),
            isin: None,
            reuters_code: None,
        },
        name: name.to_string(),
        market: normalize_kind_market(market),
        kind: infer_stock_kind(name, industry.as_deref(), main_product.as_deref()),
        industry,
        sector: None,
        main_product,
        listed_at: empty_to_none(cells.get(5).map(String::as_str).unwrap_or("")),
        fiscal_month: empty_to_none(cells.get(6).map(String::as_str).unwrap_or("")),
        homepage: empty_to_none(cells.get(8).map(String::as_str).unwrap_or("")),
        region: empty_to_none(cells.get(9).map(String::as_str).unwrap_or("")),
        source: StockSource::Kind,
        source_url: Some(KIND_UNIVERSE_URL.to_string()),
        updated_at: None,
    })
}

fn extract_cells(row: &str) -> Vec<String> {
    let mut cells = Vec::new();
    let mut rest = row;

    loop {
        let td = find_ascii_case_insensitive(rest, "<td");
        let th = find_ascii_case_insensitive(rest, "<th");
        let Some(start) = (match (td, th) {
            (Some(td), Some(th)) => Some(td.min(th)),
            (Some(td), None) => Some(td),
            (None, Some(th)) => Some(th),
            (None, None) => None,
        }) else {
            break;
        };

        let cell_start = &rest[start..];
        let Some(open_end) = cell_start.find('>') else {
            break;
        };
        let close_tag = if cell_start.to_ascii_lowercase().starts_with("<th") {
            "</th>"
        } else {
            "</td>"
        };
        let body_start = open_end + 1;
        let Some(close) = find_ascii_case_insensitive(&cell_start[body_start..], close_tag) else {
            break;
        };
        let body = &cell_start[body_start..body_start + close];
        cells.push(clean_cell(body));
        rest = &cell_start[body_start + close + close_tag.len()..];
    }

    cells
}

fn clean_cell(value: &str) -> String {
    let mut text = String::new();
    let mut inside_tag = false;

    for ch in value.chars() {
        match ch {
            '<' => inside_tag = true,
            '>' => {
                inside_tag = false;
                text.push(' ');
            }
            _ if !inside_tag => text.push(ch),
            _ => {}
        }
    }

    decode_entities(&text)
        .split_whitespace()
        .collect::<Vec<_>>()
        .join(" ")
}

fn decode_entities(value: &str) -> String {
    value
        .replace("&amp;", "&")
        .replace("&lt;", "<")
        .replace("&gt;", ">")
        .replace("&quot;", "\"")
        .replace("&#39;", "'")
        .replace("&nbsp;", " ")
}

fn find_ascii_case_insensitive(haystack: &str, needle: &str) -> Option<usize> {
    haystack
        .as_bytes()
        .windows(needle.len())
        .position(|window| window.eq_ignore_ascii_case(needle.as_bytes()))
}

fn empty_to_none(value: &str) -> Option<String> {
    let value = value.trim();
    if value.is_empty() {
        None
    } else {
        Some(value.to_string())
    }
}

fn write_cache(stocks: &[Stock], meta: &UniverseMeta) -> Result<(), String> {
    let dir = cache_dir();
    fs::create_dir_all(&dir).map_err(|err| format!("[KIND] 캐시 디렉터리 생성 실패: {}", err))?;
    write_json_atomic(&dir.join(LATEST_FILE), stocks)?;
    write_json_atomic(&dir.join(META_FILE), meta)?;
    write_json_atomic(&dir.join(format!("{}.json", meta.date)), stocks)?;
    prune_cache(false)?;
    Ok(())
}

fn write_json_atomic<T: Serialize + ?Sized>(path: &Path, value: &T) -> Result<(), String> {
    let tmp = path.with_extension("tmp");
    let json = serde_json::to_string_pretty(value)
        .map_err(|err| format!("[KIND] 캐시 JSON 직렬화 실패: {}", err))?;
    fs::write(&tmp, json).map_err(|err| format!("[KIND] 캐시 임시 파일 쓰기 실패: {}", err))?;
    fs::rename(&tmp, path).map_err(|err| format!("[KIND] 캐시 파일 교체 실패: {}", err))?;
    Ok(())
}

fn load_today_cache() -> Result<Option<UniverseSnapshot>, String> {
    let Some(snapshot) = load_any_cache()? else {
        return Ok(None);
    };
    if snapshot.meta.date == current_utc_date_string() {
        return Ok(Some(snapshot));
    }
    Ok(None)
}

fn load_any_cache() -> Result<Option<UniverseSnapshot>, String> {
    let dir = cache_dir();
    let latest_path = dir.join(LATEST_FILE);
    let meta_path = dir.join(META_FILE);
    if !latest_path.exists() || !meta_path.exists() {
        return Ok(None);
    }

    let stocks = read_json::<Vec<Stock>>(&latest_path)?;
    let meta = read_json::<UniverseMeta>(&meta_path)?;
    Ok(Some(UniverseSnapshot {
        stocks,
        meta,
        refreshed: false,
    }))
}

fn read_json<T: for<'de> Deserialize<'de>>(path: &Path) -> Result<T, String> {
    let text = fs::read_to_string(path)
        .map_err(|err| format!("[KIND] 캐시 파일 읽기 실패 ({}): {}", path.display(), err))?;
    serde_json::from_str(&text)
        .map_err(|err| format!("[KIND] 캐시 JSON 파싱 실패 ({}): {}", path.display(), err))
}

fn cache_dir() -> PathBuf {
    PathBuf::from(CACHE_DIR)
}

fn counts_by_market(stocks: &[Stock]) -> BTreeMap<String, usize> {
    let mut counts = BTreeMap::new();
    for stock in stocks {
        *counts.entry(stock.market.as_str().to_string()).or_insert(0) += 1;
    }
    counts
}

fn current_utc_date_string() -> String {
    let now_seconds = current_unix_seconds();
    let days = now_seconds.div_euclid(86_400);
    let (year, month, day) = civil_from_days(days);
    format!("{:04}-{:02}-{:02}", year, month, day)
}

fn current_utc_datetime_string() -> String {
    let now_seconds = current_unix_seconds();
    let days = now_seconds.div_euclid(86_400);
    let seconds_of_day = now_seconds.rem_euclid(86_400);
    let (year, month, day) = civil_from_days(days);
    let hour = seconds_of_day / 3_600;
    let minute = (seconds_of_day % 3_600) / 60;
    let second = seconds_of_day % 60;
    format!(
        "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}Z",
        year, month, day, hour, minute, second
    )
}

fn current_unix_seconds() -> i64 {
    SystemTime::now()
        .duration_since(UNIX_EPOCH)
        .map(|duration| duration.as_secs() as i64)
        .unwrap_or(0)
}

fn civil_from_days(days: i64) -> (i64, i64, i64) {
    let days = days + 719_468;
    let era = if days >= 0 { days } else { days - 146_096 } / 146_097;
    let day_of_era = days - era * 146_097;
    let year_of_era =
        (day_of_era - day_of_era / 1_460 + day_of_era / 36_524 - day_of_era / 146_096) / 365;
    let mut year = year_of_era + era * 400;
    let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100);
    let month_prime = (5 * day_of_year + 2) / 153;
    let day = day_of_year - (153 * month_prime + 2) / 5 + 1;
    let month = month_prime + if month_prime < 10 { 3 } else { -9 };
    year += if month <= 2 { 1 } else { 0 };

    (year, month, day)
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::core::stock::{StockKind, StockMarket};

    #[test]
    fn parses_kind_html_table() {
        let html = r#"
        <table>
          <tr><th>회사명</th><th>시장구분</th><th>종목코드</th><th>업종</th><th>주요제품</th><th>상장일</th><th>결산월</th><th>대표자명</th><th>홈페이지</th><th>지역</th></tr>
          <tr><td>삼성전자</td><td>유가</td><td>005930</td><td>통신 및 방송 장비 제조업</td><td>반도체</td><td>1975-06-11</td><td>12월</td><td>대표</td><td>https://www.samsung.com</td><td>경기도</td></tr>
          <tr><td>삼성전자우</td><td>유가</td><td>005935</td><td>통신 및 방송 장비 제조업</td><td>반도체</td><td>1989-01-01</td><td>12월</td><td>대표</td><td></td><td>경기도</td></tr>
        </table>
        "#;

        let stocks = parse_kind_html(html).unwrap();
        assert_eq!(stocks.len(), 2);
        assert_eq!(stocks[0].id.symbol, "005930");
        assert_eq!(stocks[0].market, StockMarket::Kospi);
        assert_eq!(stocks[0].kind, StockKind::CommonStock);
        assert_eq!(stocks[1].kind, StockKind::PreferredStock);
    }
}