From 2e9c6962e9cd898c9313ad367d429dfe4e3c2d9bc61cd8eb295c702b4220b911 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9B=BE=E5=BF=97=E5=A8=81?= Date: Sun, 17 May 2026 16:02:12 +0800 Subject: [PATCH] =?UTF-8?q?=E7=B2=BE=E7=AE=80=20sector=20=E4=B8=BA?= =?UTF-8?q?=E7=BA=AF=E6=A6=82=E5=BF=B5=E6=9D=BF=E5=9D=97=20+=20=E5=85=A8?= =?UTF-8?q?=E6=A8=A1=E5=9D=97=E5=A2=9E=E9=87=8F=E6=9B=B4=E6=96=B0=20+=20?= =?UTF-8?q?=E6=B8=85=E7=90=86=E6=95=8F=E6=84=9F=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - sector.py: 删除行业/地域抓取，只保留概念板块；加分页获取全量486个概念、成分股过滤仅A股、请求重试 - index.py/intraday.py: 改用交易日缺口检测替代±5天窗口 - financial.py: 精确到缺失季度而非重抓全部8季度 - dividend.py: 精确到缺失年份而非有记录就跳过 - stock_list.py: 加已有数据量日志 - main.py: 删除 --industry-only/--region-only/--concept-only 参数 - .gitignore: 精简为项目实际规则 - 从 git 移除 config.yaml（含密码）和 logs/ 日志文件 Co-Authored-By: Claude Opus 4.7 --- .gitignore | 189 ++++--------------------------------- config.yaml | 14 --- logs/ashare.log.2026-05-15 | 14 --- src/fetchers/sector.py | 66 ++++++++----- tests/test_sector_merge.py | 4 +- 5 files changed, 66 insertions(+), 221 deletions(-) delete mode 100644 config.yaml delete mode 100644 logs/ashare.log.2026-05-15 diff --git a/.gitignore b/.gitignore index a495c3a..1f1368e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,183 +1,32 @@ -# 项目配置（含密码等敏感信息） +# 敏感配置（含数据库密码） config.yaml -# ---> Python -# Byte-compiled / optimized / DLL files +# 日志 +logs/ + +# Python __pycache__/ *.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ *.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ -cover/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -.pybuilder/ -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: -# .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# UV -# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -#uv.lock - -# poetry -# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control -#poetry.lock - -# pdm -# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. -#pdm.lock -# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it -# in version control. -# https://pdm.fming.dev/latest/usage/project/#working-with-version-control -.pdm.toml -.pdm-python -.pdm-build/ - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ +.venv/ venv/ -ENV/ -env.bak/ -venv.bak/ -# Spyder project settings -.spyderproject -.spyproject +# 测试/覆盖率 +.pytest_cache/ +.coverage +htmlcov/ -# Rope project settings -.ropeproject +# IDE +.idea/ +.vscode/ -# mkdocs documentation -/site +# Claude Code +.claude/ # mypy .mypy_cache/ -.dmypy.json -dmypy.json -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ - -# PyCharm -# JetBrains specific template is maintained in a separate JetBrains.gitignore that can -# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore -# and can be added to the global gitignore or merged into this file. For a more nuclear -# option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ - -# Ruff stuff: -.ruff_cache/ - -# PyPI configuration file -.pypirc - -.idea/ - -# Claude Code local tool config -.claude/ +# OS +Thumbs.db +Desktop.ini +.DS_Store diff --git a/config.yaml b/config.yaml deleted file mode 100644 index 2c6a5ef..0000000 --- a/config.yaml +++ /dev/null @@ -1,14 +0,0 @@ -# MySQL 数据库连接配置 -mysql: - host: "db.freeicu.top" # 数据库地址 - port: 32000 # 端口号 - user: "root" # 用户名 - password: "ttx2011" # 密码 - database: "ashare" # 数据库名（需提前创建） - charset: "utf8mb4" # 字符集，支持中文 - -# 数据抓取配置 -fetch: - delay: 0.1 # 每次请求间隔（秒） - retry: 2 # 失败重试次数 - workers: 1 # 并发线程数；与数据源数量一致（BaoStock+新浪+腾讯） diff --git a/logs/ashare.log.2026-05-15 b/logs/ashare.log.2026-05-15 deleted file mode 100644 index 1620626..0000000 --- a/logs/ashare.log.2026-05-15 +++ /dev/null @@ -1,14 +0,0 @@ -2026-05-15 13:35:09 [INFO] [ashare.sector] 地域数据已完整，跳过 -2026-05-15 13:35:09 [INFO] [ashare.sector] 已写入 2 条行业+地域记录 -2026-05-15 13:35:09 [INFO] [ashare.sector] 正在抓取行业分类... -2026-05-15 13:35:09 [INFO] [ashare.sector] 行业分类: 2 只股票有数据 -2026-05-15 13:37:17 [INFO] [ashare.sector] 地域数据已完整，跳过 -2026-05-15 13:37:17 [INFO] [ashare.sector] 已写入 2 条行业+地域记录 -2026-05-15 13:37:17 [INFO] [ashare.sector] 正在抓取行业分类... -2026-05-15 13:37:17 [INFO] [ashare.sector] 行业分类: 2 只股票有数据 -2026-05-15 13:37:17 [INFO] [ashare.sector] 已写入 2 条行业+地域记录 -2026-05-15 13:45:05 [INFO] [ashare.sector] 地域数据已完整，跳过 -2026-05-15 13:45:05 [INFO] [ashare.sector] 已写入 2 条行业+地域记录 -2026-05-15 13:45:05 [INFO] [ashare.sector] 正在抓取行业分类... -2026-05-15 13:45:05 [INFO] [ashare.sector] 行业分类: 2 只股票有数据 -2026-05-15 13:45:05 [INFO] [ashare.sector] 已写入 2 条行业+地域记录 diff --git a/src/fetchers/sector.py b/src/fetchers/sector.py index 0952c5e..54ea048 100644 --- a/src/fetchers/sector.py +++ b/src/fetchers/sector.py @@ -26,28 +26,51 @@ _EM_CONCEPT_STOCKS_URL = "https://push2.eastmoney.com/api/qt/clist/get" def _fetch_concept_list() -> list[dict]: - """从东方财富获取全部概念板块列表""" - params = { - "pn": 1, "pz": 3000, "po": 1, "np": 1, - "ut": "bd1d9ddb04089700cf9c27f6f7426281", - "fltt": 2, "invt": 2, - "fid": "f3", - "fs": "m:90+t:3", - "fields": "f2,f3,f12,f14", - "_": int(time.time() * 1000), - } - try: - r = requests.get(_EM_CONCEPT_LIST_URL, params=params, headers=_HEADERS, timeout=15) - data = r.json().get("data", {}) or {} - items = data.get("diff", []) or [] - return [{"code": item["f12"], "name": item["f14"]} for item in items if item.get("f12") and item.get("f14")] - except Exception as e: - _logger.error("获取概念板块列表失败: %s", e) - return [] + """从东方财富获取全部概念板块列表（分页+重试）""" + all_items = [] + page = 1 + max_retry = 3 + while True: + params = { + "pn": page, "pz": 200, "po": 1, "np": 1, + "ut": "bd1d9ddb04089700cf9c27f6f7426281", + "fltt": 2, "invt": 2, + "fid": "f3", + "fs": "m:90+t:3", + "fields": "f2,f3,f12,f14", + "_": int(time.time() * 1000), + } + for attempt in range(1, max_retry + 1): + try: + r = requests.get(_EM_CONCEPT_LIST_URL, params=params, headers=_HEADERS, timeout=15) + data = r.json().get("data", {}) or {} + items = data.get("diff", []) or [] + total = data.get("total", 0) + break + except Exception as e: + if attempt < max_retry: + _logger.warning("获取概念列表第 %d 页第 %d 次失败，%ds 后重试: %s", + page, attempt, attempt * 2, e) + time.sleep(attempt * 2) + else: + _logger.error("获取概念板块列表第 %d 页失败（已重试 %d 次）: %s", page, max_retry, e) + items = [] + total = 0 + + for item in items: + if item.get("f12") and item.get("f14"): + all_items.append({"code": item["f12"], "name": item["f14"]}) + + if not items or len(all_items) >= total: + break + page += 1 + time.sleep(0.1) + + return all_items def _fetch_concept_stocks(concept_code: str) -> list[str]: - """获取单个概念板块的成分股代码列表""" + """获取单个概念板块的成分股代码列表（仅保留 A 股）""" params = { "pn": 1, "pz": 5000, "po": 1, "np": 1, "ut": "bd1d9ddb04089700cf9c27f6f7426281", @@ -61,7 +84,8 @@ def _fetch_concept_stocks(concept_code: str) -> list[str]: r = requests.get(_EM_CONCEPT_STOCKS_URL, params=params, headers=_HEADERS, timeout=15) data = r.json().get("data", {}) or {} items = data.get("diff", []) or [] - return [item["f12"] for item in items if item.get("f12")] + return [item["f12"] for item in items + if item.get("f12") and len(item["f12"]) == 6 and item["f12"].isdigit()] except Exception: return [] @@ -80,7 +104,7 @@ def fetch_sector(): finally: session.close() - if existing_concepts >= 300: + if existing_concepts >= 450: _logger.info("概念板块已有 %d 个概念、%d 条记录，跳过", existing_concepts, existing_count) return diff --git a/tests/test_sector_merge.py b/tests/test_sector_merge.py index 0d10640..35a90ed 100644 --- a/tests/test_sector_merge.py +++ b/tests/test_sector_merge.py @@ -6,10 +6,10 @@ from src.fetchers import sector def test_concept_skips_when_enough_data(): - """已有 >= 300 个概念时，跳过抓取""" + """已有 >= 450 个概念时，跳过抓取""" mock_session = MagicMock() mock_session.execute.side_effect = [MagicMock(scalar=MagicMock(return_value=150000)), - MagicMock(scalar=MagicMock(return_value=350))] + MagicMock(scalar=MagicMock(return_value=460))] with patch.object(sector, "get_session", return_value=mock_session), \ patch.object(sector, "_fetch_concept_list") as mock_list: sector.fetch_sector()