use crate::core::stock::{ infer_stock_kind, normalize_kind_market, validate_symbol, Stock, StockId, StockKind, StockMarket, StockSource, }; use encoding_rs::EUC_KR; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use std::fs; use std::path::{Path, PathBuf}; use std::time::{SystemTime, UNIX_EPOCH}; const KIND_UNIVERSE_URL: &str = "https://kind.krx.co.kr/corpgeneral/corpList.do?method=download&searchType=13"; const CACHE_DIR: &str = ".openstock/universe/kind"; const LATEST_FILE: &str = "latest.json"; const META_FILE: &str = "meta.json"; const SNAPSHOT_RETENTION_FILES: usize = 7; const SNAPSHOT_RETENTION_BYTES: u64 = 25 * 1024 * 1024; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct UniverseMeta { pub source: String, pub source_url: String, pub date: String, pub refreshed_at: String, pub stock_count: usize, pub counts_by_market: BTreeMap, } #[derive(Debug, Clone)] pub struct UniverseSnapshot { pub stocks: Vec, pub meta: UniverseMeta, pub refreshed: bool, } pub fn load_or_refresh(force: bool) -> Result { if !force { if let Some(snapshot) = load_today_cache()? { return Ok(snapshot); } } match refresh() { Ok(snapshot) => Ok(snapshot), Err(err) => { if let Some(snapshot) = load_any_cache()? { return Ok(UniverseSnapshot { refreshed: false, ..snapshot }); } Err(err) } } } pub fn refresh() -> Result { let body = download()?; let html = decode_euc_kr(&body); let mut stocks = parse_kind_html(&html)?; stocks.sort_by(|a, b| { a.market .cmp(&b.market) .then_with(|| a.kind.cmp(&b.kind)) .then_with(|| a.id.symbol.cmp(&b.id.symbol)) }); let today = current_utc_date_string(); for stock in &mut stocks { stock.updated_at = Some(today.clone()); } let meta = UniverseMeta { source: "KIND".to_string(), source_url: KIND_UNIVERSE_URL.to_string(), date: today.clone(), refreshed_at: current_utc_datetime_string(), stock_count: stocks.len(), counts_by_market: counts_by_market(&stocks), }; write_cache(&stocks, &meta)?; Ok(UniverseSnapshot { stocks, meta, refreshed: true, }) } pub fn load_cached() -> Result, String> { load_any_cache() } pub fn prune_cache(dry_run: bool) -> Result { crate::core::cache::prune_json_snapshots( "universe/kind", &cache_dir(), &[LATEST_FILE, META_FILE], SNAPSHOT_RETENTION_FILES, SNAPSHOT_RETENTION_BYTES, dry_run, ) } pub fn chunk_stocks(stocks: &[Stock], size: usize) -> Vec { let size = size.max(1); let mut groups = BTreeMap::<(StockMarket, StockKind), Vec<&Stock>>::new(); for stock in stocks { groups .entry((stock.market, stock.kind)) .or_default() .push(stock); } let mut chunks = Vec::new(); for ((market, kind), group) in groups { for (index, chunk) in group.chunks(size).enumerate() { let first = chunk.first().copied(); let last = chunk.last().copied(); chunks.push(serde_json::json!({ "chunk_id": format!("{}:{}:{:04}", market.as_str(), kind.as_str(), index + 1), "index": index + 1, "size": size, "count": chunk.len(), "start_symbol": first.map(|stock| stock.id.symbol.as_str()), "end_symbol": last.map(|stock| stock.id.symbol.as_str()), "market": market.as_str(), "kind": kind.as_str(), })); } } chunks } fn download() -> Result, String> { let response = crate::core::http::agent() .get(KIND_UNIVERSE_URL) .header("User-Agent", "openstock/0.1") .call() .map_err(|err| format!("[KIND] 상장법인목록 다운로드 실패: {}", err))?; let status = response.status(); let body = response .into_body() .read_to_vec() .map_err(|err| format!("[KIND] 상장법인목록 응답 읽기 실패: {}", err))?; if !status.is_success() { return Err(format!( "[KIND] 상장법인목록 다운로드 오류 ({}): {} bytes", status, body.len() )); } Ok(body) } fn decode_euc_kr(bytes: &[u8]) -> String { let (decoded, _, _) = EUC_KR.decode(bytes); decoded.into_owned() } fn parse_kind_html(html: &str) -> Result, String> { let mut rows = Vec::new(); let mut rest = html; while let Some(start) = find_ascii_case_insensitive(rest, "') else { break; }; let row_body_start = open_end + 1; let Some(close) = find_ascii_case_insensitive(&row_start[row_body_start..], "") else { break; }; let row_body = &row_start[row_body_start..row_body_start + close]; let cells = extract_cells(row_body); if cells.len() >= 10 && cells[0] != "회사명" { if let Some(stock) = row_to_stock(&cells) { rows.push(stock); } } rest = &row_start[row_body_start + close + "".len()..]; } if rows.is_empty() { return Err("[KIND] 상장법인목록에서 종목을 찾지 못했습니다.".to_string()); } Ok(rows) } fn row_to_stock(cells: &[String]) -> Option { let name = cells.first()?.trim(); let market = cells.get(1)?.trim(); let symbol = cells.get(2)?.trim(); let industry = empty_to_none(cells.get(3).map(String::as_str).unwrap_or("")); let main_product = empty_to_none(cells.get(4).map(String::as_str).unwrap_or("")); if name.is_empty() || validate_symbol(symbol).is_err() { return None; } Some(Stock { id: StockId { symbol: symbol.to_string(), isin: None, reuters_code: None, }, name: name.to_string(), market: normalize_kind_market(market), kind: infer_stock_kind(name, industry.as_deref(), main_product.as_deref()), industry, sector: None, main_product, listed_at: empty_to_none(cells.get(5).map(String::as_str).unwrap_or("")), fiscal_month: empty_to_none(cells.get(6).map(String::as_str).unwrap_or("")), homepage: empty_to_none(cells.get(8).map(String::as_str).unwrap_or("")), region: empty_to_none(cells.get(9).map(String::as_str).unwrap_or("")), source: StockSource::Kind, source_url: Some(KIND_UNIVERSE_URL.to_string()), updated_at: None, }) } fn extract_cells(row: &str) -> Vec { let mut cells = Vec::new(); let mut rest = row; loop { let td = find_ascii_case_insensitive(rest, " Some(td.min(th)), (Some(td), None) => Some(td), (None, Some(th)) => Some(th), (None, None) => None, }) else { break; }; let cell_start = &rest[start..]; let Some(open_end) = cell_start.find('>') else { break; }; let close_tag = if cell_start.to_ascii_lowercase().starts_with("" } else { "" }; let body_start = open_end + 1; let Some(close) = find_ascii_case_insensitive(&cell_start[body_start..], close_tag) else { break; }; let body = &cell_start[body_start..body_start + close]; cells.push(clean_cell(body)); rest = &cell_start[body_start + close + close_tag.len()..]; } cells } fn clean_cell(value: &str) -> String { let mut text = String::new(); let mut inside_tag = false; for ch in value.chars() { match ch { '<' => inside_tag = true, '>' => { inside_tag = false; text.push(' '); } _ if !inside_tag => text.push(ch), _ => {} } } decode_entities(&text) .split_whitespace() .collect::>() .join(" ") } fn decode_entities(value: &str) -> String { value .replace("&", "&") .replace("<", "<") .replace(">", ">") .replace(""", "\"") .replace("'", "'") .replace(" ", " ") } fn find_ascii_case_insensitive(haystack: &str, needle: &str) -> Option { haystack .as_bytes() .windows(needle.len()) .position(|window| window.eq_ignore_ascii_case(needle.as_bytes())) } fn empty_to_none(value: &str) -> Option { let value = value.trim(); if value.is_empty() { None } else { Some(value.to_string()) } } fn write_cache(stocks: &[Stock], meta: &UniverseMeta) -> Result<(), String> { let dir = cache_dir(); fs::create_dir_all(&dir).map_err(|err| format!("[KIND] 캐시 디렉터리 생성 실패: {}", err))?; write_json_atomic(&dir.join(LATEST_FILE), stocks)?; write_json_atomic(&dir.join(META_FILE), meta)?; write_json_atomic(&dir.join(format!("{}.json", meta.date)), stocks)?; prune_cache(false)?; Ok(()) } fn write_json_atomic(path: &Path, value: &T) -> Result<(), String> { let tmp = path.with_extension("tmp"); let json = serde_json::to_string_pretty(value) .map_err(|err| format!("[KIND] 캐시 JSON 직렬화 실패: {}", err))?; fs::write(&tmp, json).map_err(|err| format!("[KIND] 캐시 임시 파일 쓰기 실패: {}", err))?; fs::rename(&tmp, path).map_err(|err| format!("[KIND] 캐시 파일 교체 실패: {}", err))?; Ok(()) } fn load_today_cache() -> Result, String> { let Some(snapshot) = load_any_cache()? else { return Ok(None); }; if snapshot.meta.date == current_utc_date_string() { return Ok(Some(snapshot)); } Ok(None) } fn load_any_cache() -> Result, String> { let dir = cache_dir(); let latest_path = dir.join(LATEST_FILE); let meta_path = dir.join(META_FILE); if !latest_path.exists() || !meta_path.exists() { return Ok(None); } let stocks = read_json::>(&latest_path)?; let meta = read_json::(&meta_path)?; Ok(Some(UniverseSnapshot { stocks, meta, refreshed: false, })) } fn read_json Deserialize<'de>>(path: &Path) -> Result { let text = fs::read_to_string(path) .map_err(|err| format!("[KIND] 캐시 파일 읽기 실패 ({}): {}", path.display(), err))?; serde_json::from_str(&text) .map_err(|err| format!("[KIND] 캐시 JSON 파싱 실패 ({}): {}", path.display(), err)) } fn cache_dir() -> PathBuf { PathBuf::from(CACHE_DIR) } fn counts_by_market(stocks: &[Stock]) -> BTreeMap { let mut counts = BTreeMap::new(); for stock in stocks { *counts.entry(stock.market.as_str().to_string()).or_insert(0) += 1; } counts } fn current_utc_date_string() -> String { let now_seconds = current_unix_seconds(); let days = now_seconds.div_euclid(86_400); let (year, month, day) = civil_from_days(days); format!("{:04}-{:02}-{:02}", year, month, day) } fn current_utc_datetime_string() -> String { let now_seconds = current_unix_seconds(); let days = now_seconds.div_euclid(86_400); let seconds_of_day = now_seconds.rem_euclid(86_400); let (year, month, day) = civil_from_days(days); let hour = seconds_of_day / 3_600; let minute = (seconds_of_day % 3_600) / 60; let second = seconds_of_day % 60; format!( "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}Z", year, month, day, hour, minute, second ) } fn current_unix_seconds() -> i64 { SystemTime::now() .duration_since(UNIX_EPOCH) .map(|duration| duration.as_secs() as i64) .unwrap_or(0) } fn civil_from_days(days: i64) -> (i64, i64, i64) { let days = days + 719_468; let era = if days >= 0 { days } else { days - 146_096 } / 146_097; let day_of_era = days - era * 146_097; let year_of_era = (day_of_era - day_of_era / 1_460 + day_of_era / 36_524 - day_of_era / 146_096) / 365; let mut year = year_of_era + era * 400; let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100); let month_prime = (5 * day_of_year + 2) / 153; let day = day_of_year - (153 * month_prime + 2) / 5 + 1; let month = month_prime + if month_prime < 10 { 3 } else { -9 }; year += if month <= 2 { 1 } else { 0 }; (year, month, day) } #[cfg(test)] mod tests { use super::*; use crate::core::stock::{StockKind, StockMarket}; #[test] fn parses_kind_html_table() { let html = r#"
회사명시장구분종목코드업종주요제품상장일결산월대표자명홈페이지지역
삼성전자유가005930통신 및 방송 장비 제조업반도체1975-06-1112월대표https://www.samsung.com경기도
삼성전자우유가005935통신 및 방송 장비 제조업반도체1989-01-0112월대표경기도
"#; let stocks = parse_kind_html(html).unwrap(); assert_eq!(stocks.len(), 2); assert_eq!(stocks[0].id.symbol, "005930"); assert_eq!(stocks[0].market, StockMarket::Kospi); assert_eq!(stocks[0].kind, StockKind::CommonStock); assert_eq!(stocks[1].kind, StockKind::PreferredStock); } }