zoe/ashare-data
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

init

Browse Source
This commit is contained in:
曾志威
2026-05-10 01:00:51 +08:00
parent 217b12c205
commit feef7b63d0
10 changed files with 88 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.idea/.gitignore
Generated
-10
View File
@@ -1,10 +0,0 @@
# 默认忽略的文件
/shelf/
/workspace.xml
# 基于编辑器的 HTTP 客户端请求
/httpRequests/
# 已忽略包含查询文件的默认文件夹
/queries/
# Datasource local storage ignored files
/dataSources/
/dataSources.local.xml
.idea/ashare-data.iml
Generated
-8
View File
@@ -1,8 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>
.idea/claudeCodeTabState.xml
Generated
-21
View File
@@ -1,21 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ClaudeCodeTabState">
<option name="tabSessions">
<map>
<entry key="0">
<value>
<TabSessionState>
<option name="provider" value="claude" />
<option name="sessionId" value="fff69755-9aa0-4879-bbe1-5e61efee9bea" />
<option name="cwd" value="$PROJECT_DIR$" />
<option name="model" value="claude-opus-4-7[1m]" />
<option name="permissionMode" value="bypassPermissions" />
<option name="reasoningEffort" value="high" />
</TabSessionState>
</value>
</entry>
</map>
</option>
</component>
</project>
.idea/inspectionProfiles/profiles_settings.xml
Generated
-6
View File
@@ -1,6 +0,0 @@
<component name="InspectionProjectProfileManager">
<settings>
<option name="USE_PROJECT_PROFILE" value="false" />
<version value="1.0" />
</settings>
</component>
.idea/modules.xml
Generated
-8
View File
@@ -1,8 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/ashare-data.iml" filepath="$PROJECT_DIR$/.idea/ashare-data.iml" />
</modules>
</component>
</project>
config.yaml
+1 -1
View File
@@ -11,4 +11,4 @@ mysql:
fetch:
delay: 0.1 # 每次请求间隔（秒）
retry: 2 # 失败重试次数
workers: 5 # 并发线程数；3个数据源(BaoStock+新浪+腾讯)，建议 5-8
workers: 3 # 并发线程数；与数据源数量一致（BaoStock+新浪+腾讯）
src/db.py
+3
View File
@@ -262,6 +262,9 @@ def batch_upsert(model_cls: type[Base], rows: list[dict], index_columns: list[st
}
if update_dict:
stmt = stmt.on_duplicate_key_update(**update_dict)
else:
# 无可更新列时（如 TradingDay 只有唯一键列），用唯一键本身做 no-op 更新
stmt = stmt.on_duplicate_key_update(**{index_columns[0]: stmt.inserted[index_columns[0]]})
session.execute(stmt)
session.commit()
except Exception as e:
src/fetchers/daily.py
+1 -1
View File
@@ -295,7 +295,7 @@ def _fetch_one(code: str, start_date: str, end_date: str, source: str, datalen:
def fetch_daily(start_date: str | None = None, end_date: str | None = None):
cfg = get_fetch_config()
delay = cfg.get("delay", 0.1)
workers = cfg.get("workers", 5)
workers = cfg.get("workers", 3)
codes = _get_stock_codes()
if not codes:
src/fetchers/trading_day.py
+76 -25
View File
@@ -1,19 +1,74 @@
"""交易日历模块 — 提供三级回退策略获取交易日列表
"""交易日历模块 — 提供多级回退策略获取交易日列表
1. 本地 trading_day 表（最快，之前已缓存）
2. AKShare 新浪交易日历接口（从交易所获取官方日历）
3. 从 stock_daily 表已有数据推断（最后手段）
2. AKShare 新浪交易日历接口（数据准确，覆盖 1990-12-19 ~ 2026-12-31）
3. BaoStock 交易日历（备用，需过滤掉周末）
4. 从 stock_daily 表已有数据推断（最后手段）
交易日历是日线数据完整性判断的关键依据：
股票在某个日期范围内的记录数必须等于交易日数才算完整。
"""
import time
from datetime import datetime
import baostock as bs
import akshare as ak
from src.db import TradingDay, batch_upsert, get_session
from sqlalchemy import select, func
def _fetch_akshare(sd: str, ed: str) -> list[str] | None:
"""从 AKShare 新浪接口获取交易日历（数据准确，覆盖1990~2026）"""
try:
df = ak.tool_trade_date_hist_sina()
df["date_str"] = df["trade_date"].astype(str)
mask = (df["date_str"] >= sd) & (df["date_str"] <= ed)
filtered = df.loc[mask]
days = filtered["date_str"].tolist()
return days if days else None
except Exception:
return None
def _fetch_baostock(sd: str, ed: str) -> list[str] | None:
"""从 BaoStock 获取交易日历，过滤掉周末（BaoStock 会返回包括周末在内的所有日期）"""
try:
bs.login()
rs = bs.query_trade_dates(start_date=sd, end_date=ed)
days = []
while (rs.error_code == "0") and rs.next():
d = rs.get_row_data()[0]
# 过滤周末：weekday() 返回 0-6 (Mon-Sun)
if datetime.strptime(d, "%Y-%m-%d").weekday() < 5:
days.append(d)
bs.logout()
return days if days else None
except Exception:
try:
bs.logout()
except Exception:
pass
return None
def _fetch_and_save(start_date: str, end_date: str) -> list[str]:
"""按优先级尝试数据源，获取交易日并保存到本地表"""
sd = f"{start_date[:4]}-{start_date[4:6]}-{start_date[6:8]}"
ed = f"{end_date[:4]}-{end_date[4:6]}-{end_date[6:8]}"
for name, fn in [("AKShare", _fetch_akshare), ("BaoStock", _fetch_baostock)]:
print(f" 正在从 {name} 获取交易日历 {sd} ~ {ed}...", flush=True)
days = fn(sd, ed)
if days:
rows = [{"date": d} for d in days]
batch_upsert(TradingDay, rows, ["date"])
print(f" 交易日历已保存，{len(days)} 个交易日", flush=True)
return days
print(" 所有数据源均失败，将从已有行情数据推断", flush=True)
return _infer_from_daily(start_date, end_date)
def get_trading_days(start_date: str, end_date: str) -> list[str]:
"""获取指定范围内的交易日列表，本地表优先，缺失则从数据源拉取"""
session = get_session()
@@ -30,28 +85,7 @@ def get_trading_days(start_date: str, end_date: str) -> list[str]:
finally:
session.close()
# 本地无数据，从 AKShare 拉取交易所交易日历
print(f" 正在从交易所获取交易日历 {start_date} ~ {end_date}...", flush=True)
try:
df = ak.tool_trade_date_hist_sina()
# AKShare 返回的 trade_date 列是 datetime.date 对象，不能直接和字符串比较
# 必须先转为字符串再做范围过滤
sd = f"{start_date[:4]}-{start_date[4:6]}-{start_date[6:8]}"
ed = f"{end_date[:4]}-{end_date[4:6]}-{end_date[6:8]}"
df["date_str"] = df["trade_date"].astype(str)
mask = (df["date_str"] >= sd) & (df["date_str"] <= ed)
filtered = df.loc[mask]
rows = [{"date": row["date_str"]} for _, row in filtered.iterrows()]
if rows:
batch_upsert(TradingDay, rows, ["date"])
print(f" 交易日历已保存，{len(rows)} 个交易日", flush=True)
return [r["date"] for r in rows]
except Exception as e:
print(f" 获取交易日历失败: {e}，将从已有行情数据推断", flush=True)
# 回退：从 stock_daily 推断
return _infer_from_daily(start_date, end_date)
return _fetch_and_save(start_date, end_date)
def _infer_from_daily(start_date: str, end_date: str) -> list[str]:
@@ -68,3 +102,20 @@ def _infer_from_daily(start_date: str, end_date: str) -> list[str]:
return [str(row[0]) for row in result]
finally:
session.close()
def fetch_trading_days(start_date: str | None = None, end_date: str | None = None):
"""独立抓取交易日历并保存到 trading_day 表
用法：python -m src.main --trading-day --start-date 19901219 --end-date 20261231
默认从 1990-12-19（沪市开市日）到今天，建议一次性拉取完整日历。
"""
if end_date is None:
end_date = time.strftime("%Y%m%d")
if start_date is None:
start_date = "19901219"
print(f"正在抓取交易日历 {start_date} ~ {end_date}...", flush=True)
days = _fetch_and_save(start_date, end_date)
if not days:
print(" 无交易日数据", flush=True)
src/main.py
+7 -1
View File
@@ -2,6 +2,7 @@
用法示例：
python -m src.main --stock-list # 先抓取股票列表
python -m src.main --trading-day --start-date 20200101 --end-date 20261231
python -m src.main --daily --start-date 20260501 --end-date 20260508
python -m src.main --all # 全量抓取
python -m src.main --financial --symbol 000001 # 单只股票财务数据
@@ -17,6 +18,7 @@ from src.db import init_db
def main():
parser = argparse.ArgumentParser(description="A股数据抓取工具")
parser.add_argument("--stock-list", action="store_true", help="抓取股票列表")
parser.add_argument("--trading-day", action="store_true", help="抓取交易日历")
parser.add_argument("--daily", action="store_true", help="抓取日线行情")
parser.add_argument("--financial", action="store_true", help="抓取财务报表")
parser.add_argument("--money-flow", action="store_true", help="抓取资金流向")
@@ -30,7 +32,7 @@ def main():
args = parser.parse_args()
if not any([args.stock_list, args.daily, args.financial, args.money_flow,
if not any([args.stock_list, args.trading_day, args.daily, args.financial, args.money_flow,
args.dragon_tiger, args.dividend, args.intraday, args.all]):
parser.print_help()
return
@@ -44,6 +46,10 @@ def main():
from src.fetchers.stock_list import fetch_stock_list
fetch_stock_list()
if args.all or args.trading_day:
from src.fetchers.trading_day import fetch_trading_days
fetch_trading_days(start_date=args.start_date, end_date=args.end_date)
if args.all or args.daily:
from src.fetchers.daily import fetch_daily
fetch_daily(start_date=args.start_date, end_date=args.end_date)