提交 b7e2693e authored 作者: zhangqianwei's avatar zhangqianwei

ljy_收集 5~10 个 GitHub 热门仓库 README + 目录结构

上级 395b530b
import sys
from pathlib import Path
from typing import Any
try:
from github_client import GitHubClient
from schemas import RawRepoRecord, RepoCandidate
except ImportError:
CURRENT_DIR = Path(__file__).resolve().parent
if str(CURRENT_DIR) not in sys.path:
sys.path.insert(0, str(CURRENT_DIR))
from github_client import GitHubClient
from schemas import RawRepoRecord, RepoCandidate
def build_api_source(owner: str, repo: str, default_branch: str) -> dict[str, str]:
return {
"repo_api": f"https://api.github.com/repos/{owner}/{repo}",
"readme_api": f"https://api.github.com/repos/{owner}/{repo}/readme",
"tree_api": (
f"https://api.github.com/repos/{owner}/{repo}/git/trees/"
f"{default_branch}?recursive=1"
),
}
def _extract_tree_items(tree_payload: dict[str, Any]) -> list[dict[str, Any]]:
items = tree_payload.get("tree", [])
if not isinstance(items, list):
return []
return [item for item in items if isinstance(item, dict)]
def collect_single_repo(candidate: RepoCandidate, client: GitHubClient | None = None) -> RawRepoRecord:
github_client = client or GitHubClient()
owner = candidate["owner"]
repo = candidate["repo"]
repo_meta = github_client.get_repo_meta(owner, repo)
default_branch = str(repo_meta.get("default_branch") or "main")
readme_payload = github_client.get_readme(owner, repo)
tree_payload = github_client.get_repo_tree(owner, repo, default_branch)
return RawRepoRecord(
repo_name=str(repo_meta.get("name") or repo),
owner=owner,
repo_url=str(repo_meta.get("html_url") or candidate["repo_url"]),
description=str(repo_meta.get("description") or ""),
stars=int(repo_meta.get("stargazers_count") or 0),
forks=int(repo_meta.get("forks_count") or 0),
watchers=int(repo_meta.get("watchers_count") or 0),
primary_language=str(repo_meta.get("language") or ""),
topics=list(repo_meta.get("topics") or []),
default_branch=default_branch,
updated_at=str(repo_meta.get("updated_at") or ""),
readme_text=str(readme_payload.get("content") or ""),
tree_items=_extract_tree_items(tree_payload),
api_source=build_api_source(owner, repo, default_branch),
)
from pathlib import Path
BASE_DIR = Path(__file__).resolve().parents[2]
DATA_DIR = BASE_DIR / "shared" / "data" / "github_repos"
GITHUB_API_BASE = "https://api.github.com"
REQUEST_TIMEOUT = 20
RAW_OUTPUT_DIR = DATA_DIR / "raw"
NORMALIZED_OUTPUT_DIR = DATA_DIR / "normalized"
LOG_OUTPUT_DIR = DATA_DIR / "logs"
CANDIDATES_FILE = DATA_DIR / "candidates.json"
import json
import os
import sys
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from urllib.error import HTTPError, URLError
from urllib.parse import quote, urlencode
from urllib.request import Request, urlopen
try:
from config import GITHUB_API_BASE, REQUEST_TIMEOUT
except ImportError:
CURRENT_DIR = Path(__file__).resolve().parent
if str(CURRENT_DIR) not in sys.path:
sys.path.insert(0, str(CURRENT_DIR))
from config import GITHUB_API_BASE, REQUEST_TIMEOUT
class GitHubApiError(RuntimeError):
"""Raised when GitHub API requests fail."""
@dataclass(slots=True)
class GitHubClient:
token: str | None = None
api_base: str = GITHUB_API_BASE
timeout: int = REQUEST_TIMEOUT
def __post_init__(self) -> None:
if self.token is None:
self.token = os.getenv("GITHUB_TOKEN") or os.getenv("GITHUB_API_TOKEN")
def _build_headers(self) -> dict[str, str]:
headers = {
"Accept": "application/vnd.github+json",
"User-Agent": "studyweb-repo-ingestion",
"X-GitHub-Api-Version": "2022-11-28",
}
if self.token:
headers["Authorization"] = f"Bearer {self.token}"
return headers
def _request_json(self, url: str) -> dict[str, Any]:
request = Request(url=url, headers=self._build_headers(), method="GET")
try:
with urlopen(request, timeout=self.timeout) as response:
payload = response.read().decode("utf-8")
except HTTPError as exc:
message = exc.read().decode("utf-8", errors="replace")
raise GitHubApiError(
f"GitHub API request failed with status {exc.code}: {message}"
) from exc
except URLError as exc:
raise GitHubApiError(f"GitHub API connection failed: {exc.reason}") from exc
try:
return json.loads(payload)
except json.JSONDecodeError as exc:
raise GitHubApiError("GitHub API returned invalid JSON.") from exc
def get_repo_meta(self, owner: str, repo: str) -> dict[str, Any]:
owner_name = quote(owner)
repo_name = quote(repo)
url = f"{self.api_base}/repos/{owner_name}/{repo_name}"
return self._request_json(url)
def get_readme(self, owner: str, repo: str) -> dict[str, Any]:
owner_name = quote(owner)
repo_name = quote(repo)
url = f"{self.api_base}/repos/{owner_name}/{repo_name}/readme"
return self._request_json(url)
def get_repo_tree(self, owner: str, repo: str, ref: str) -> dict[str, Any]:
owner_name = quote(owner)
repo_name = quote(repo)
encoded_ref = quote(ref)
url = (
f"{self.api_base}/repos/{owner_name}/{repo_name}/git/trees/"
f"{encoded_ref}?recursive=1"
)
return self._request_json(url)
def search_repositories(
self,
query: str,
sort: str = "stars",
order: str = "desc",
per_page: int = 10,
page: int = 1,
) -> dict[str, Any]:
params = urlencode(
{
"q": query,
"sort": sort,
"order": order,
"per_page": per_page,
"page": page,
}
)
url = f"{self.api_base}/search/repositories?{params}"
return self._request_json(url)
from typing import Any
def build_knowledge_json_schema() -> dict[str, Any]:
return {
"title": "repo_knowledge_extraction",
"type": "object",
"additionalProperties": False,
"required": [
"repo_name",
"repo_url",
"repo_category",
"summary",
"learning_value",
"prerequisites",
"knowledge_points",
"suggested_learning_path",
"relevant_directories",
"tags",
],
"properties": {
"repo_name": {"type": "string"},
"repo_url": {"type": "string"},
"repo_category": {"type": "string"},
"summary": {"type": "string"},
"learning_value": {"type": "string"},
"prerequisites": {
"type": "array",
"items": {"type": "string"},
},
"knowledge_points": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": False,
"required": [
"name",
"category",
"description",
"importance",
"evidence",
],
"properties": {
"name": {"type": "string"},
"category": {"type": "string"},
"description": {"type": "string"},
"importance": {"type": "string"},
"evidence": {"type": "string"},
},
},
},
"suggested_learning_path": {
"type": "array",
"items": {"type": "string"},
},
"relevant_directories": {
"type": "array",
"items": {"type": "string"},
},
"tags": {
"type": "array",
"items": {"type": "string"},
},
},
}
def validate_knowledge_record(record: dict[str, Any]) -> None:
required_fields = {
"repo_name": str,
"repo_url": str,
"repo_category": str,
"summary": str,
"learning_value": str,
"prerequisites": list,
"knowledge_points": list,
"suggested_learning_path": list,
"relevant_directories": list,
"tags": list,
}
for field_name, expected_type in required_fields.items():
if field_name not in record:
raise ValueError(f"Missing required field: {field_name}")
if not isinstance(record[field_name], expected_type):
raise TypeError(f"Field {field_name} must be {expected_type.__name__}")
for field_name in ("prerequisites", "suggested_learning_path", "relevant_directories", "tags"):
if not all(isinstance(item, str) for item in record[field_name]):
raise TypeError(f"All items in {field_name} must be strings")
for item in record["knowledge_points"]:
if not isinstance(item, dict):
raise TypeError("Each knowledge point must be an object")
for key in ("name", "category", "description", "importance", "evidence"):
if key not in item or not isinstance(item[key], str) or not item[key].strip():
raise ValueError(f"Invalid knowledge point field: {key}")
import base64
import importlib.util
import re
import sys
from pathlib import Path
from typing import Any
try:
from schemas import NormalizedRepoMetadata, NormalizedRepoRecord, RawRepoRecord
except ImportError:
CURRENT_DIR = Path(__file__).resolve().parent
if str(CURRENT_DIR) not in sys.path:
sys.path.insert(0, str(CURRENT_DIR))
schema_spec = importlib.util.spec_from_file_location(
"repo_ingestion_schemas",
CURRENT_DIR / "schemas.py",
)
if schema_spec is None or schema_spec.loader is None:
raise ImportError("Unable to load local schemas.py")
schema_module = importlib.util.module_from_spec(schema_spec)
schema_spec.loader.exec_module(schema_module)
NormalizedRepoMetadata = schema_module.NormalizedRepoMetadata
NormalizedRepoRecord = schema_module.NormalizedRepoRecord
RawRepoRecord = schema_module.RawRepoRecord
MAX_TREE_LINES = 300
MAX_MAIN_FEATURES = 8
FEATURE_SECTION_KEYWORDS = (
"feature",
"features",
"highlight",
"highlights",
"key features",
"why choose streamlit?",
)
INLINE_FEATURE_MARKERS = (
"features:",
"features are:",
"key features are:",
"high-level features:",
"following components:",
"used either as:",
"why choose",
)
GENERIC_HEADINGS = {
"installation",
"install",
"usage",
"quickstart",
"quick start",
"getting started",
"contributing",
"license",
"community",
"documentation",
"docs",
}
NOISY_HEADINGS = {
"sponsors",
"opinions",
"fastapi conf",
"fastapi mini documentary",
"resources",
"community cloud",
"get inspired",
"contribute",
"license",
"requirements",
"example",
"example upgrade",
"what is streamlit?",
}
NOISY_FEATURES = {
"sponsors",
"opinions",
"fastapi conf",
"fastapi mini documentary",
"resources",
"community cloud",
"requirements",
"example",
"example upgrade",
}
def decode_readme_content(encoded_content: str) -> str:
if not encoded_content:
return ""
compact_content = "".join(encoded_content.splitlines())
try:
decoded_bytes = base64.b64decode(compact_content, validate=False)
except Exception:
return encoded_content
return decoded_bytes.decode("utf-8", errors="replace").strip()
def _insert_path(tree: dict[str, Any], path: str, item_type: str) -> None:
parts = [part for part in path.split("/") if part]
if not parts:
return
current = tree
for part in parts[:-1]:
current = current.setdefault(part, {})
current[parts[-1]] = {} if item_type == "tree" else None
def _render_tree_lines(tree: dict[str, Any], prefix: str = "") -> list[str]:
lines: list[str] = []
keys = sorted(tree.keys(), key=lambda name: (tree[name] is None, name.lower()))
for index, key in enumerate(keys):
is_last = index == len(keys) - 1
connector = "└── " if is_last else "├── "
lines.append(f"{prefix}{connector}{key}")
child = tree[key]
if isinstance(child, dict) and child:
child_prefix = f"{prefix}{' ' if is_last else '│ '}"
lines.extend(_render_tree_lines(child, child_prefix))
return lines
def build_directory_tree(tree_items: list[dict[str, Any]], max_lines: int = MAX_TREE_LINES) -> str:
if not tree_items:
return ""
tree: dict[str, Any] = {}
for item in tree_items:
path = str(item.get("path") or "").strip()
item_type = str(item.get("type") or "blob").strip()
if not path:
continue
_insert_path(tree, path, item_type)
lines = _render_tree_lines(tree)
if not lines:
return ""
if len(lines) > max_lines:
remaining = len(lines) - max_lines
lines = lines[:max_lines]
lines.append(f"... ({remaining} more entries omitted)")
return "\n".join(lines)
def _normalize_feature_text(text: str) -> str:
cleaned = text.strip()
cleaned = re.sub(r"^[\-\*\+\d\.\)\s]+", "", cleaned)
cleaned = re.sub(r"\[([^\]]+)\]\([^)]+\)", r"\1", cleaned)
cleaned = re.sub(r"<[^>]+>", "", cleaned)
cleaned = re.sub(r"\*+", "", cleaned)
cleaned = re.sub(r"`+", "", cleaned)
cleaned = re.sub(r"\s+", " ", cleaned)
return cleaned.strip(" :.-")
def _is_noisy_feature(feature: str) -> bool:
normalized = feature.lower().strip(" .:-")
if normalized in NOISY_FEATURES:
return True
if len(normalized) < 6:
return True
if normalized.startswith("what is "):
return True
return False
def _extract_intro_sentences(readme_text: str, topics: list[str], limit: int) -> list[str]:
features: list[str] = []
paragraphs = [part.strip() for part in readme_text.split("\n\n") if part.strip()]
for paragraph in paragraphs[:12]:
if paragraph.startswith("#") or paragraph.startswith("##"):
continue
if paragraph.startswith(".. ") or paragraph.startswith("|"):
continue
if "image::" in paragraph or ":target:" in paragraph:
continue
cleaned = _normalize_feature_text(paragraph)
if not cleaned or len(cleaned) < 40:
continue
if "installation" in cleaned.lower() or "contributing" in cleaned.lower():
continue
if cleaned.count("|") >= 2:
continue
first_sentence = re.split(r"(?<=[.!?])\s+", cleaned, maxsplit=1)[0].strip()
if first_sentence and first_sentence not in features:
features.append(first_sentence[:180])
if len(features) >= limit:
return features
for topic in topics:
topic_text = topic.replace("-", " ").strip()
if not topic_text:
continue
candidate = topic_text[:1].upper() + topic_text[1:]
if candidate not in features:
features.append(candidate)
if len(features) >= limit:
break
return features
def extract_main_features(readme_text: str, topics: list[str] | None = None, limit: int = MAX_MAIN_FEATURES) -> list[str]:
if not readme_text:
return []
topic_list = topics or []
lines = readme_text.splitlines()
features: list[str] = []
collecting = False
for raw_line in lines:
line = raw_line.strip()
lower_line = line.lower().strip("# ").strip()
if line.startswith("#"):
collecting = lower_line in FEATURE_SECTION_KEYWORDS
continue
if any(marker in lower_line for marker in INLINE_FEATURE_MARKERS):
collecting = True
continue
if collecting and re.match(r"^[-*+]\s+.+", line):
candidate = _normalize_feature_text(line)
if candidate and not _is_noisy_feature(candidate) and candidate not in features:
features.append(candidate)
if len(features) >= limit:
return features
continue
if collecting and line:
collecting = False
if features:
return features
for raw_line in lines:
line = raw_line.strip()
if not line.startswith("## "):
continue
heading = _normalize_feature_text(line.lstrip("#"))
if not heading:
continue
lower_heading = heading.lower()
if lower_heading in GENERIC_HEADINGS or lower_heading in NOISY_HEADINGS:
continue
if heading not in features:
features.append(heading)
if len(features) >= limit:
break
if features:
return features
return _extract_intro_sentences(readme_text, topic_list, limit)
def normalize_repo_record(raw_record: RawRepoRecord) -> NormalizedRepoRecord:
readme_text = decode_readme_content(raw_record["readme_text"])
metadata = NormalizedRepoMetadata(
owner=raw_record["owner"],
default_branch=raw_record["default_branch"],
topics=list(raw_record["topics"]),
)
return NormalizedRepoRecord(
repo_name=raw_record["repo_name"],
repo_url=raw_record["repo_url"],
stars=raw_record["stars"],
primary_language=raw_record["primary_language"],
updated_at=raw_record["updated_at"],
readme_text=readme_text,
directory_tree=build_directory_tree(raw_record["tree_items"]),
main_features=extract_main_features(readme_text, topics=list(raw_record["topics"])),
metadata=metadata,
)
import json
import re
import sys
from pathlib import Path
from typing import Any
try:
from collector import collect_single_repo
from config import CANDIDATES_FILE, NORMALIZED_OUTPUT_DIR, RAW_OUTPUT_DIR
from parser import normalize_repo_record
from schemas import NormalizedRepoRecord, RawRepoRecord, RepoCandidate
except ImportError:
CURRENT_DIR = Path(__file__).resolve().parent
if str(CURRENT_DIR) not in sys.path:
sys.path.insert(0, str(CURRENT_DIR))
from collector import collect_single_repo
from config import CANDIDATES_FILE, NORMALIZED_OUTPUT_DIR, RAW_OUTPUT_DIR
from parser import normalize_repo_record
from schemas import NormalizedRepoRecord, RawRepoRecord, RepoCandidate
def ensure_output_dirs() -> None:
RAW_OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
NORMALIZED_OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
def load_candidates(candidates_file: Path = CANDIDATES_FILE) -> list[RepoCandidate]:
payload = json.loads(candidates_file.read_text(encoding="utf-8"))
if not isinstance(payload, list):
raise ValueError("candidates.json must contain a list.")
return [candidate for candidate in payload if isinstance(candidate, dict)]
def build_output_stem(owner: str, repo: str) -> str:
raw_stem = f"{owner}__{repo}".strip().lower()
return re.sub(r"[^a-z0-9._-]+", "-", raw_stem)
def write_json(path: Path, payload: dict[str, Any]) -> None:
path.write_text(
json.dumps(payload, ensure_ascii=False, indent=2),
encoding="utf-8",
)
def process_candidate(
candidate: RepoCandidate,
client: Any = None,
) -> tuple[RawRepoRecord, NormalizedRepoRecord]:
raw_record = collect_single_repo(candidate, client=client)
normalized_record = normalize_repo_record(raw_record)
return raw_record, normalized_record
def write_repo_outputs(raw_record: RawRepoRecord, normalized_record: NormalizedRepoRecord) -> tuple[Path, Path]:
ensure_output_dirs()
stem = build_output_stem(raw_record["owner"], raw_record["repo_name"])
raw_path = RAW_OUTPUT_DIR / f"{stem}.json"
normalized_path = NORMALIZED_OUTPUT_DIR / f"{stem}.json"
write_json(raw_path, raw_record)
write_json(normalized_path, normalized_record)
return raw_path, normalized_path
def run_batch_collect(
candidates: list[RepoCandidate],
client: Any = None,
) -> list[dict[str, str]]:
results: list[dict[str, str]] = []
for candidate in candidates:
raw_record, normalized_record = process_candidate(candidate, client=client)
raw_path, normalized_path = write_repo_outputs(raw_record, normalized_record)
results.append(
{
"repo_name": raw_record["repo_name"],
"raw_path": str(raw_path),
"normalized_path": str(normalized_path),
}
)
return results
import json
import re
from typing import Any
MAX_README_CHARS = 12000
MAX_TREE_LINES = 120
def _strip_markdown_noise(text: str) -> str:
cleaned = re.sub(r"!\[[^\]]*\]\([^)]+\)", "", text)
cleaned = re.sub(r"<img[^>]*>", "", cleaned)
cleaned = re.sub(r"<picture>.*?</picture>", "", cleaned, flags=re.DOTALL)
cleaned = re.sub(r"`{3}.*?`{3}", "", cleaned, flags=re.DOTALL)
cleaned = re.sub(r"\n{3,}", "\n\n", cleaned)
return cleaned.strip()
def build_prompt_context(normalized_record: dict[str, Any]) -> dict[str, Any]:
readme_text = _strip_markdown_noise(str(normalized_record.get("readme_text") or ""))
directory_tree = str(normalized_record.get("directory_tree") or "")
tree_lines = directory_tree.splitlines()[:MAX_TREE_LINES]
return {
"repo_name": normalized_record.get("repo_name", ""),
"repo_url": normalized_record.get("repo_url", ""),
"primary_language": normalized_record.get("primary_language", ""),
"stars": normalized_record.get("stars", 0),
"updated_at": normalized_record.get("updated_at", ""),
"main_features": normalized_record.get("main_features", []),
"metadata": normalized_record.get("metadata", {}),
"directory_tree_excerpt": "\n".join(tree_lines),
"readme_excerpt": readme_text[:MAX_README_CHARS],
}
def build_system_prompt() -> str:
return (
"你是一个负责分析 GitHub 开源仓库的技术研究助手。"
"你的任务是从 README、目录结构和仓库元信息中,提取适合后续培训、知识整理和项目分析使用的结构化知识点。"
"输出必须严格符合给定 JSON Schema,不要输出额外解释,不要输出 Markdown。"
"如果某个信息不能完全确定,可以基于 README 和目录结构做保守推断,但不要编造与仓库内容明显不符的信息。"
)
def build_user_prompt(normalized_record: dict[str, Any]) -> str:
context = build_prompt_context(normalized_record)
return (
"请基于下面的 GitHub 仓库信息,提取结构化知识点。\n\n"
"目标要求:\n"
"1. 判断仓库类别,例如深度学习框架、Web 框架、LLM 应用框架、开发工具、数据平台等。\n"
"2. 总结仓库的核心定位和学习价值。\n"
"3. 提炼 5 到 8 个值得学习的知识点,每个知识点都要附带来源证据。\n"
"4. 给出建议的前置知识和学习路径。\n"
"5. 标出最值得关注的目录。\n\n"
f"仓库名称:{context['repo_name']}\n"
f"仓库链接:{context['repo_url']}\n"
f"主要语言:{context['primary_language']}\n"
f"Stars:{context['stars']}\n"
f"最近更新时间:{context['updated_at']}\n"
f"main_features:{json.dumps(context['main_features'], ensure_ascii=False)}\n"
f"metadata:{json.dumps(context['metadata'], ensure_ascii=False)}\n\n"
"目录结构摘录:\n"
f"{context['directory_tree_excerpt']}\n\n"
"README 摘录:\n"
f"{context['readme_excerpt']}\n"
)
def build_prompt_packet(normalized_record: dict[str, Any], schema: dict[str, Any]) -> dict[str, Any]:
return {
"repo_name": normalized_record.get("repo_name", ""),
"repo_url": normalized_record.get("repo_url", ""),
"messages": [
{"role": "system", "content": build_system_prompt()},
{"role": "user", "content": build_user_prompt(normalized_record)},
],
"json_schema": schema,
}
import json
import re
import sys
from pathlib import Path
from typing import Any
try:
from github_client import GitHubClient
from schemas import RepoCandidate, RepoRecommendation, RepoSearchRequest
except ImportError:
CURRENT_DIR = Path(__file__).resolve().parent
if str(CURRENT_DIR) not in sys.path:
sys.path.insert(0, str(CURRENT_DIR))
from github_client import GitHubClient
from schemas import RepoCandidate, RepoRecommendation, RepoSearchRequest
DEFAULT_LIMIT = 10
DEFAULT_MIN_STARS = 100
MAX_GITHUB_SEARCH_PAGE_SIZE = 100
LOW_RELEVANCE_HINTS = {
"awesome",
"course",
"demo",
"example",
"examples",
"learn",
"sample",
"samples",
"template",
"tutorial",
"workshop",
}
def normalize_keywords(raw_keywords: list[str] | str) -> list[str]:
if isinstance(raw_keywords, str):
pieces = re.split(r"[,,\s]+", raw_keywords)
else:
pieces = []
for keyword in raw_keywords:
pieces.extend(re.split(r"[,,\s]+", str(keyword)))
result: list[str] = []
for piece in pieces:
keyword = piece.strip().lower()
if keyword and keyword not in result:
result.append(keyword)
return result
def build_github_search_query(request: RepoSearchRequest) -> str:
keywords = normalize_keywords(request.get("keywords", []))
if not keywords:
raise ValueError("At least one keyword is required for GitHub repository search.")
query_parts = keywords[:]
language = str(request.get("language") or "").strip()
if language:
query_parts.append(f"language:{language}")
min_stars = int(request.get("min_stars") or DEFAULT_MIN_STARS)
if min_stars > 0:
query_parts.append(f"stars:>{min_stars}")
updated_after = str(request.get("updated_after") or "").strip()
if updated_after:
query_parts.append(f"pushed:>{updated_after}")
query_parts.append("archived:false")
return " ".join(query_parts)
def _keyword_hit_count(text: str, keywords: list[str]) -> int:
lower_text = text.lower()
return sum(1 for keyword in keywords if keyword in lower_text)
def _semantic_hit_count(repo_item: dict[str, Any], keywords: list[str]) -> int:
semantic_text = " ".join(
[
str(repo_item.get("description") or ""),
" ".join(repo_item.get("topics") or []),
]
)
return _keyword_hit_count(semantic_text, keywords)
def _required_semantic_hits(keywords: list[str]) -> int:
if len(keywords) <= 1:
return 1
return min(2, len(keywords))
def _low_relevance_hint_hits(repo_item: dict[str, Any], keywords: list[str]) -> list[str]:
if any(keyword in LOW_RELEVANCE_HINTS for keyword in keywords):
return []
text = " ".join(
[
str(repo_item.get("name") or ""),
str(repo_item.get("full_name") or ""),
str(repo_item.get("description") or ""),
" ".join(repo_item.get("topics") or []),
]
).lower()
tokens = {token for token in re.split(r"[^a-z0-9]+", text) if token}
return sorted(LOW_RELEVANCE_HINTS & tokens)
def _is_semantically_relevant(repo_item: dict[str, Any], keywords: list[str]) -> bool:
return _semantic_hit_count(repo_item, keywords) >= _required_semantic_hits(keywords)
def _score_repo(repo_item: dict[str, Any], keywords: list[str], language: str) -> tuple[float, list[str]]:
name_text = " ".join(
[
str(repo_item.get("name") or ""),
str(repo_item.get("full_name") or ""),
]
)
description_text = str(repo_item.get("description") or "")
topics = repo_item.get("topics") or []
topics_text = " ".join(topics)
name_hits = _keyword_hit_count(name_text, keywords)
description_hits = _keyword_hit_count(description_text, keywords)
topic_hits = _keyword_hit_count(topics_text, keywords)
semantic_hits = _keyword_hit_count(f"{description_text} {topics_text}", keywords)
keyword_count = max(len(keywords), 1)
semantic_score = min(semantic_hits / keyword_count, 1.0) * 45
description_score = min(description_hits / keyword_count, 1.0) * 15
topic_score = min(topic_hits / keyword_count, 1.0) * 20
name_score = min(name_hits / keyword_count, 1.0) * 5
repo_language = str(repo_item.get("language") or "")
language_score = 10 if language and repo_language.lower() == language.lower() else 0
stars = int(repo_item.get("stargazers_count") or 0)
stars_score = min(stars / 20000, 1.0) * 10
low_relevance_hints = _low_relevance_hint_hits(repo_item, keywords)
low_relevance_penalty = 15 if low_relevance_hints else 0
score = round(
max(
semantic_score
+ description_score
+ topic_score
+ name_score
+ language_score
+ stars_score
- low_relevance_penalty,
0,
),
2,
)
reasons: list[str] = []
if semantic_hits:
reasons.append(f"仓库描述或 topics 命中 {semantic_hits} 个关键词")
if name_hits:
reasons.append(f"仓库名称命中 {name_hits} 个关键词")
if language_score:
reasons.append(f"主要语言为 {repo_language},符合语言筛选条件")
if stars:
reasons.append(f"GitHub stars 为 {stars},作为辅助排序因素")
if topic_hits:
reasons.append("仓库 topics 与搜索关键词存在交集")
if low_relevance_hints:
reasons.append(f"检测到偏教程或示例类信号,已降权:{', '.join(low_relevance_hints[:3])}")
if repo_item.get("updated_at"):
reasons.append(f"最近更新时间:{repo_item['updated_at']}")
return score, reasons
def _to_owner_repo(repo_item: dict[str, Any]) -> tuple[str, str]:
full_name = str(repo_item.get("full_name") or "")
if "/" not in full_name:
raise ValueError(f"Invalid GitHub full_name: {full_name}")
owner, repo = full_name.split("/", maxsplit=1)
return owner, repo
def search_repositories(
request: RepoSearchRequest,
client: GitHubClient | None = None,
) -> list[RepoRecommendation]:
github_client = client or GitHubClient()
limit = int(request.get("limit") or DEFAULT_LIMIT)
per_page = min(max(limit * 4, 20), MAX_GITHUB_SEARCH_PAGE_SIZE)
query = build_github_search_query(request)
payload = github_client.search_repositories(query=query, per_page=per_page)
items = payload.get("items", [])
if not isinstance(items, list):
return []
keywords = normalize_keywords(request.get("keywords", []))
language = str(request.get("language") or "")
recommendations: list[RepoRecommendation] = []
for item in items:
if not isinstance(item, dict):
continue
if item.get("fork") or item.get("archived"):
continue
if not _is_semantically_relevant(item, keywords):
continue
owner, repo = _to_owner_repo(item)
score, reasons = _score_repo(item, keywords, language)
recommendations.append(
RepoRecommendation(
owner=owner,
repo=repo,
repo_url=str(item.get("html_url") or f"https://github.com/{owner}/{repo}"),
description=str(item.get("description") or ""),
stars=int(item.get("stargazers_count") or 0),
primary_language=str(item.get("language") or ""),
topics=list(item.get("topics") or []),
updated_at=str(item.get("updated_at") or ""),
match_score=score,
recommend_reasons=reasons,
)
)
recommendations.sort(key=lambda item: item["match_score"], reverse=True)
return recommendations[:limit]
def recommendations_to_candidates(recommendations: list[RepoRecommendation]) -> list[RepoCandidate]:
return [
RepoCandidate(
owner=item["owner"],
repo=item["repo"],
repo_url=item["repo_url"],
)
for item in recommendations
]
def write_search_outputs(
recommendations: list[RepoRecommendation],
output_dir: Path,
stem: str,
) -> tuple[Path, Path]:
output_dir.mkdir(parents=True, exist_ok=True)
recommendations_path = output_dir / f"{stem}_recommendations.json"
candidates_path = output_dir / f"{stem}_candidates.json"
recommendations_path.write_text(
json.dumps(recommendations, ensure_ascii=False, indent=2),
encoding="utf-8",
)
candidates_path.write_text(
json.dumps(recommendations_to_candidates(recommendations), ensure_ascii=False, indent=2),
encoding="utf-8",
)
return recommendations_path, candidates_path
import argparse
import json
import sys
from pathlib import Path
try:
from pipeline import load_candidates, run_batch_collect
except ImportError:
CURRENT_DIR = Path(__file__).resolve().parent
if str(CURRENT_DIR) not in sys.path:
sys.path.insert(0, str(CURRENT_DIR))
from pipeline import load_candidates, run_batch_collect
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Collect GitHub repo README and tree data.")
parser.add_argument(
"--limit",
type=int,
default=1,
help="Number of repositories to collect from candidates.json.",
)
parser.add_argument(
"--candidates-file",
default="",
help="Optional candidates JSON file generated by run_search.py.",
)
return parser.parse_args()
def main() -> int:
args = parse_args()
candidates_file = Path(args.candidates_file) if args.candidates_file else None
candidates = load_candidates(candidates_file) if candidates_file else load_candidates()
if args.limit > 0:
candidates = candidates[: args.limit]
results = run_batch_collect(candidates)
print(json.dumps(results, ensure_ascii=False, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())
import json
import sys
from pathlib import Path
try:
from knowledge_pipeline import run_prepare_knowledge_packets
except ImportError:
CURRENT_DIR = Path(__file__).resolve().parent
if str(CURRENT_DIR) not in sys.path:
sys.path.insert(0, str(CURRENT_DIR))
from knowledge_pipeline import run_prepare_knowledge_packets
def main() -> int:
results = run_prepare_knowledge_packets()
print(json.dumps(results, ensure_ascii=False, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())
import argparse
import json
import re
import sys
from pathlib import Path
try:
from config import DATA_DIR
from recommender import search_repositories, write_search_outputs
from schemas import RepoSearchRequest
except ImportError:
CURRENT_DIR = Path(__file__).resolve().parent
if str(CURRENT_DIR) not in sys.path:
sys.path.insert(0, str(CURRENT_DIR))
from config import DATA_DIR
from recommender import search_repositories, write_search_outputs
from schemas import RepoSearchRequest
def _build_stem(keywords: str, language: str) -> str:
raw = "_".join(part for part in [keywords, language] if part).lower()
return re.sub(r"[^a-z0-9._-]+", "-", raw).strip("-") or "github_search"
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Search GitHub repositories by keywords.")
parser.add_argument("--keywords", required=True, help="Search keywords, e.g. 'rag harness'.")
parser.add_argument("--language", default="", help="Optional GitHub language filter.")
parser.add_argument("--min-stars", type=int, default=100, help="Minimum GitHub stars.")
parser.add_argument("--updated-after", default="", help="Optional pushed-after date, e.g. 2025-01-01.")
parser.add_argument("--limit", type=int, default=10, help="Number of repositories to recommend.")
parser.add_argument("--write", action="store_true", help="Write recommendations and candidates JSON files.")
return parser.parse_args()
def main() -> int:
args = parse_args()
request = RepoSearchRequest(
keywords=[args.keywords],
language=args.language,
min_stars=args.min_stars,
updated_after=args.updated_after,
limit=args.limit,
)
recommendations = search_repositories(request)
result: dict[str, object] = {"recommendations": recommendations}
if args.write:
stem = _build_stem(args.keywords, args.language)
output_dir = DATA_DIR / "search_results"
recommendations_path, candidates_path = write_search_outputs(recommendations, output_dir, stem)
result["recommendations_path"] = str(recommendations_path)
result["candidates_path"] = str(candidates_path)
print(json.dumps(result, ensure_ascii=False, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())
from typing import Any, TypedDict
class RepoCandidate(TypedDict):
owner: str
repo: str
repo_url: str
class RepoSearchRequest(TypedDict, total=False):
keywords: list[str]
language: str
min_stars: int
updated_after: str
limit: int
class RepoRecommendation(TypedDict):
owner: str
repo: str
repo_url: str
description: str
stars: int
primary_language: str
topics: list[str]
updated_at: str
match_score: float
recommend_reasons: list[str]
class RepoApiSource(TypedDict):
repo_api: str
readme_api: str
tree_api: str
class RawRepoRecord(TypedDict):
repo_name: str
owner: str
repo_url: str
description: str
stars: int
forks: int
watchers: int
primary_language: str
topics: list[str]
default_branch: str
updated_at: str
readme_text: str
tree_items: list[dict[str, Any]]
api_source: RepoApiSource
class NormalizedRepoMetadata(TypedDict):
owner: str
default_branch: str
topics: list[str]
class NormalizedRepoRecord(TypedDict):
repo_name: str
repo_url: str
stars: int
primary_language: str
updated_at: str
readme_text: str
directory_tree: str
main_features: list[str]
metadata: NormalizedRepoMetadata
# 今日任务规划:GitHub 热门仓库 README 与目录结构采集
> **日期**:2026-07-13
> **目标**:今天完成 5~10 个 GitHub 热门开源仓库的筛选与采集,拿到 README、目录结构、仓库基础信息,并为后续 `Prompt + JSON Schema` 结构化处理准备统一数据格式。
> **验收标准**:完成 5~10 个高质量仓库筛选;每个仓库具备 README、目录结构、stars、语言、更新时间、仓库链接;采集逻辑放入 `ai-service/`;采集结果统一落到 `shared/data/github_repos/`。
---
## 一、任务拆解
| 序号 | 任务项 | 说明 |
| --- | --- | --- |
| 1 | 明确筛选标准 | 定义“高质量热门仓库”的筛选条件 |
| 2 | 确定候选仓库池 | 先列出 10 个左右候选仓库 |
| 3 | 目录规划 | 明确代码落点和数据落点 |
| 4 | GitHub API 验证 | 验证仓库基础信息、README、目录结构是否可抓取 |
| 5 | 统一输出格式 | 明确 JSON 字段结构,避免后续返工 |
| 6 | 采集脚本实现 | 编写最小可用采集脚本 |
| 7 | 采集 5~10 个仓库 | 执行实际采集并保存结果 |
| 8 | 数据检查 | 校验字段是否完整、内容是否可用 |
| 9 | 结果整理 | 输出候选仓库清单与采集结果说明 |
| 10 | 后续接口准备 | 为 Prompt + JSON Schema 预留结构化输入格式 |
---
## 二、今日目录规划
按当前仓库约定执行:
```text
/home/admin2/studyweb/
├─ ai-service/
│ └─ repo_ingestion/
│ ├─ README.md
│ ├─ __init__.py
│ ├─ config.py
│ ├─ github_client.py
│ ├─ collector.py
│ ├─ parser.py
│ ├─ schemas.py
│ ├─ pipeline.py
│ └─ run_collect.py
├─ shared/
└─ data/
└─ github_repos/
├─ candidates.json
├─ raw/
├─ normalized/
└─ logs/
```
说明:
- `ai-service/`:放采集逻辑和后续 AI 处理入口
- `shared/data/github_repos/`:放采集结果
- `raw/`:原始抓取结果
- `normalized/`:统一结构后的结果,便于后续 Prompt / JSON Schema 使用
---
## 三、执行步骤
### 第一步:明确筛选标准
```text
□ 仓库必须为公开开源仓库
□ stars 需达到基本门槛
□ README 相对完整,不能只有几行
□ 目录结构清晰,不能完全不可读
□ 最近仍有更新,避免太旧的项目
□ 技术方向尽量和平台后续分析目标相关
```
### 第二步:确定候选仓库池
```text
□ 先列出 10 个左右候选仓库
□ 记录仓库名、链接、stars、语言、更新时间
□ 优先选 README 清晰、目录结构规整的仓库
□ 最终收敛到 5~10 个样本
```
### 第三步:统一输出格式
每个仓库建议至少包含以下字段:
```json
{
"repo_name": "",
"repo_url": "",
"stars": 0,
"primary_language": "",
"updated_at": "",
"readme_text": "",
"directory_tree": "",
"main_features": []
}
```
要求:
```text
□ 字段命名统一
□ README 保留原文
□ 目录结构尽量转成可读树形文本
□ 主要功能点先允许为空,后续可由 Prompt 结构化提取
```
### 第四步:GitHub API 验证
```text
□ 确认可以拿到仓库基础信息
□ 确认可以拿到 README 内容
□ 确认可以拿到目录结构相关数据
□ 确认 API 调用方式和限流情况
```
优先验证:
```text
1. 仓库基础信息接口
2. README 接口
3. 仓库树/目录结构接口
```
### 第五步:采集脚本实现
```text
□ 在 ai-service/repo_ingestion/ 下实现 GitHub API 调用逻辑
□ 将 README、目录结构、仓库元信息统一组装
□ 输出 raw JSON
□ 再做一层 normalized JSON
```
建议最小实现顺序:
```text
□ github_client.py:封装 API 请求
□ collector.py:按仓库清单批量采集
□ parser.py:把目录数据转成可读结构
□ schemas.py:定义统一输出字段
```
### 第六步:采集实际仓库样本
```text
□ 至少先跑通 1~2 个仓库,验证流程
□ 确认字段完整后,再扩到 5~10 个仓库
□ 原始结果落到 shared/data/github_repos/raw/
□ 标准化结果落到 shared/data/github_repos/normalized/
□ 候选仓库清单保存在 candidates.json
```
### 第七步:结果检查
```text
□ 每个仓库必须有 repo_url
□ 每个仓库必须有 stars
□ 每个仓库必须有 primary_language
□ 每个仓库必须有 updated_at
□ 每个仓库必须有 README
□ 每个仓库必须有目录结构
□ JSON 结构保持统一
```
### 第八步:为 Prompt + JSON Schema 做准备
```text
□ 保证 README 原文可直接作为 Prompt 输入
□ 保证目录结构字段可直接作为上下文补充
□ 保证 normalized 结果字段稳定,方便后续 JSON Schema 约束
□ main_features 字段预留给后续结构化抽取
```
---
## 四、验收检查单
```text
□ 已完成 5~10 个 GitHub 热门仓库筛选
□ 每个仓库的 README 已采集
□ 每个仓库的目录结构已采集
□ 每个仓库的 stars / language / updated_at / repo_url 已记录
□ 采集逻辑已放入 ai-service/
□ 采集结果已落到 shared/data/github_repos/
□ 原始结果与标准化结果已区分
□ 输出结构可直接供 Prompt + JSON Schema 使用
```
---
## 五、参考路径
- 平台架构文档:`/home/admin2/studyweb/平台文档/架构大纲.md`
- 今日任务规划文件:`/home/admin2/studyweb/ljy_docs/7-13+任务.md`
- 采集逻辑建议目录:`/home/admin2/studyweb/ai-service/repo_ingestion/`
- 采集结果建议目录:`/home/admin2/studyweb/shared/data/github_repos/`
---
## 六、若遇到阻塞
```text
□ GitHub API 限流 -> 检查是否需要 Token,降低请求频率
□ README 获取失败 -> 检查默认分支和 README 文件路径
□ 目录结构不完整 -> 检查 tree 接口返回层级和递归参数
□ 仓库太大 -> 先保留关键目录结构,不追求一次抓全全部细节
□ 字段不统一 -> 先收敛 schema,再批量重跑
□ main_features 难以人工总结 -> 先保留原始 README,后续交给 Prompt 提取
```
[
{
"owner": "pytorch",
"repo": "pytorch",
"repo_url": "https://github.com/pytorch/pytorch"
},
{
"owner": "fastapi",
"repo": "fastapi",
"repo_url": "https://github.com/fastapi/fastapi"
},
{
"owner": "langchain-ai",
"repo": "langchain",
"repo_url": "https://github.com/langchain-ai/langchain"
},
{
"owner": "microsoft",
"repo": "vscode",
"repo_url": "https://github.com/microsoft/vscode"
},
{
"owner": "huggingface",
"repo": "transformers",
"repo_url": "https://github.com/huggingface/transformers"
},
{
"owner": "scikit-learn",
"repo": "scikit-learn",
"repo_url": "https://github.com/scikit-learn/scikit-learn"
},
{
"owner": "streamlit",
"repo": "streamlit",
"repo_url": "https://github.com/streamlit/streamlit"
},
{
"owner": "vllm-project",
"repo": "vllm",
"repo_url": "https://github.com/vllm-project/vllm"
},
{
"owner": "run-llama",
"repo": "llama_index",
"repo_url": "https://github.com/run-llama/llama_index"
},
{
"owner": "apache",
"repo": "airflow",
"repo_url": "https://github.com/apache/airflow"
}
]
{
"repo_name": "airflow",
"repo_url": "https://github.com/apache/airflow",
"repo_category": "数据与工作流平台",
"summary": "Apache Airflow (or simply Airflow) is a platform to programmatically author, schedule, and monitor workflows.",
"learning_value": "airflow 适合作为 数据与工作流平台 的学习样本,可以帮助理解 Python 生态下的核心能力、工程组织方式以及真实项目的功能拆分。",
"prerequisites": [
"Python 基础",
"任务编排基础",
"数据工程基本概念"
],
"knowledge_points": [
{
"name": "Project Focus",
"category": "核心能力",
"description": "README 将这一项作为核心能力直接强调,适合作为理解仓库定位的第一优先点。",
"importance": "高",
"evidence": "README main_features: Project Focus"
},
{
"name": "Principles",
"category": "核心能力",
"description": "README 将这一项作为核心能力直接强调,适合作为理解仓库定位的第一优先点。",
"importance": "高",
"evidence": "README main_features: Principles"
},
{
"name": "Installing from PyPI",
"category": "核心能力",
"description": "README 将这一项作为核心能力直接强调,适合作为理解仓库定位的第一优先点。",
"importance": "高",
"evidence": "README main_features: Installing from PyPI"
},
{
"name": "Official source code",
"category": "核心能力",
"description": "README 将这一项作为核心能力直接强调,适合作为理解仓库定位的第一优先点。",
"importance": "高",
"evidence": "README main_features: Official source code"
},
{
"name": "Convenience packages",
"category": "核心能力",
"description": "README 将这一项作为核心能力直接强调,适合作为理解仓库定位的第一优先点。",
"importance": "高",
"evidence": "README main_features: Convenience packages"
},
{
"name": "User Interface",
"category": "核心能力",
"description": "README 将这一项作为核心能力直接强调,适合作为理解仓库定位的第一优先点。",
"importance": "高",
"evidence": "README main_features: User Interface"
},
{
"name": "Semantic versioning",
"category": "核心能力",
"description": "README 将这一项作为核心能力直接强调,适合作为理解仓库定位的第一优先点。",
"importance": "高",
"evidence": "README main_features: Semantic versioning"
},
{
"name": "Version Life Cycle",
"category": "核心能力",
"description": "README 将这一项作为核心能力直接强调,适合作为理解仓库定位的第一优先点。",
"importance": "高",
"evidence": "README main_features: Version Life Cycle"
}
],
"suggested_learning_path": [
"先通读 README,明确项目解决的问题和核心定位",
"再结合目录结构定位入口文件、核心模块和示例代码",
"最后挑 1 到 2 个代表性模块深入阅读实现细节",
"建议优先阅读目录:docs, scripts, airflow-core"
],
"relevant_directories": [
"docs",
"scripts",
"airflow-core",
"airflow-ctl",
"airflow-ctl-tests",
"airflow-e2e-tests",
"chart",
"clients"
],
"tags": [
"Python",
"airflow",
"apache",
"apache-airflow",
"automation",
"dag",
"data-engineering",
"data-integration"
]
}
\ No newline at end of file
{
"repo_name": "fastapi",
"repo_url": "https://github.com/fastapi/fastapi",
"repo_category": "Web 框架",
"summary": "FastAPI is a modern, fast (high-performance), web framework for building APIs with Python based on standard Python type hints.",
"learning_value": "fastapi 适合作为 Web 框架 的学习样本,可以帮助理解 Python 生态下的核心能力、工程组织方式以及真实项目的功能拆分。",
"prerequisites": [
"Python 基础",
"HTTP 与 REST 基础",
"后端服务开发基础"
],
"knowledge_points": [
{
"name": "高性能",
"category": "核心能力",
"description": "仓库强调这一能力,重点在于:Very high performance, on par with NodeJS and Go (thanks to Starlette and Pydantic). One of the fastest Python frameworks available",
"importance": "高",
"evidence": "README main_features: Fast: Very high performance, on par with NodeJS and Go (thanks to Starlette and Pydantic). One of the fastest Python frameworks available"
},
{
"name": "开发效率",
"category": "核心能力",
"description": "仓库强调这一能力,重点在于:Increase the speed to develop features by about 200% to 300%",
"importance": "高",
"evidence": "README main_features: Fast to code: Increase the speed to develop features by about 200% to 300%"
},
{
"name": "降低错误率",
"category": "核心能力",
"description": "仓库强调这一能力,重点在于:Reduce about 40% of human (developer) induced errors",
"importance": "高",
"evidence": "README main_features: Fewer bugs: Reduce about 40% of human (developer) induced errors"
},
{
"name": "编辑器友好",
"category": "核心能力",
"description": "仓库强调这一能力,重点在于:Great editor support. Completion everywhere. Less time debugging",
"importance": "高",
"evidence": "README main_features: Intuitive: Great editor support. Completion everywhere. Less time debugging"
},
{
"name": "低学习成本",
"category": "核心能力",
"description": "仓库强调这一能力,重点在于:Designed to be easy to use and learn. Less time reading docs",
"importance": "高",
"evidence": "README main_features: Easy: Designed to be easy to use and learn. Less time reading docs"
},
{
"name": "低样板代码",
"category": "核心能力",
"description": "仓库强调这一能力,重点在于:Minimize code duplication. Multiple features from each parameter declaration. Fewer bugs",
"importance": "高",
"evidence": "README main_features: Short: Minimize code duplication. Multiple features from each parameter declaration. Fewer bugs"
},
{
"name": "生产可用性",
"category": "核心能力",
"description": "仓库强调这一能力,重点在于:Get production-ready code. With automatic interactive documentation",
"importance": "高",
"evidence": "README main_features: Robust: Get production-ready code. With automatic interactive documentation"
},
{
"name": "标准兼容性",
"category": "核心能力",
"description": "仓库强调这一能力,重点在于:Based on (and fully compatible with) the open standards for APIs: OpenAPI (previously known as Swagger) and JSON Schema",
"importance": "高",
"evidence": "README main_features: Standards-based: Based on (and fully compatible with) the open standards for APIs: OpenAPI (previously known as Swagger) and JSON Schema"
}
],
"suggested_learning_path": [
"先通读 README,明确项目解决的问题和核心定位",
"再结合目录结构定位入口文件、核心模块和示例代码",
"最后挑 1 到 2 个代表性模块深入阅读实现细节",
"重点关注路由、请求处理、数据校验和文档生成相关模块",
"建议优先阅读目录:fastapi, docs, docs_src"
],
"relevant_directories": [
"fastapi",
"docs",
"docs_src",
"tests",
"scripts"
],
"tags": [
"Python",
"api",
"async",
"asyncio",
"fastapi",
"framework",
"json",
"json-schema"
]
}
\ No newline at end of file
{
"repo_name": "transformers",
"repo_url": "https://github.com/huggingface/transformers",
"repo_category": "LLM 应用框架",
"summary": "Transformers acts as the model-definition framework for state-of-the-art machine learning with text, computer vision, audio, video, and multimodal models, for both inference and training.",
"learning_value": "transformers 适合作为 LLM 应用框架 的学习样本,可以帮助理解 Python 生态下的核心能力、工程组织方式以及真实项目的功能拆分。",
"prerequisites": [
"Python 基础",
"Prompt 基础",
"向量检索或 RAG 基础"
],
"knowledge_points": [
{
"name": "音频模型能力",
"category": "多模态模型",
"description": "仓库主题标签明确指向“音频模型能力”方向,说明这部分能力在项目定位中占比较高,阅读时值得优先关注。",
"importance": "中",
"evidence": "metadata.topics contains: audio"
},
{
"name": "深度学习",
"category": "深度学习",
"description": "仓库主题标签明确指向“深度学习”方向,说明这部分能力在项目定位中占比较高,阅读时值得优先关注。",
"importance": "中",
"evidence": "metadata.topics contains: deep-learning"
},
{
"name": "大模型应用",
"category": "大模型应用",
"description": "仓库主题标签明确指向“大模型应用”方向,说明这部分能力在项目定位中占比较高,阅读时值得优先关注。",
"importance": "中",
"evidence": "metadata.topics contains: llm"
},
{
"name": "机器学习",
"category": "机器学习",
"description": "仓库主题标签明确指向“机器学习”方向,说明这部分能力在项目定位中占比较高,阅读时值得优先关注。",
"importance": "中",
"evidence": "metadata.topics contains: machine-learning"
},
{
"name": "模型仓库生态",
"category": "LLM 应用框架",
"description": "仓库主题标签明确指向“模型仓库生态”方向,说明这部分能力在项目定位中占比较高,阅读时值得优先关注。",
"importance": "中",
"evidence": "metadata.topics contains: model-hub"
},
{
"name": "阅读 src 目录",
"category": "核心源码",
"description": "src 目录通常承载与 核心源码 相关的关键实现,适合作为进入源码细节时的优先切入点。",
"importance": "中",
"evidence": "directory_tree contains top-level entry: src"
},
{
"name": "阅读 docs 目录",
"category": "文档体系",
"description": "docs 目录通常承载与 文档体系 相关的关键实现,适合作为进入源码细节时的优先切入点。",
"importance": "中",
"evidence": "directory_tree contains top-level entry: docs"
},
{
"name": "阅读 examples 目录",
"category": "示例用法",
"description": "examples 目录通常承载与 示例用法 相关的关键实现,适合作为进入源码细节时的优先切入点。",
"importance": "中",
"evidence": "directory_tree contains top-level entry: examples"
}
],
"suggested_learning_path": [
"先通读 README,明确项目解决的问题和核心定位",
"再结合目录结构定位入口文件、核心模块和示例代码",
"最后挑 1 到 2 个代表性模块深入阅读实现细节",
"重点关注 Prompt、检索、工具调用或 Agent 相关模块",
"建议优先阅读目录:src, docs, examples"
],
"relevant_directories": [
"src",
"docs",
"examples",
"tests",
"scripts",
"benchmark",
"benchmark_v2",
"docker"
],
"tags": [
"Python",
"audio",
"deep-learning",
"deepseek",
"gemma",
"glm",
"hacktoberfest",
"llm"
]
}
\ No newline at end of file
{
"repo_name": "langchain",
"repo_url": "https://github.com/langchain-ai/langchain",
"repo_category": "LLM 应用框架",
"summary": "LangChain is a framework for building agents and LLM-powered applications. It helps you chain together interoperable components and third-party integrations to simplify AI application development — all while future-proofing decisions as the underlying technology evolves.",
"learning_value": "langchain 适合作为 LLM 应用框架 的学习样本,可以帮助理解 Python 生态下的核心能力、工程组织方式以及真实项目的功能拆分。",
"prerequisites": [
"Python 基础",
"Prompt 基础",
"向量检索或 RAG 基础"
],
"knowledge_points": [
{
"name": "LangChain ecosystem",
"category": "核心能力",
"description": "README 将这一项作为核心能力直接强调,适合作为理解仓库定位的第一优先点。",
"importance": "高",
"evidence": "README main_features: LangChain ecosystem"
},
{
"name": "Agent 机制",
"category": "Agent 机制",
"description": "仓库主题标签明确指向“Agent 机制”方向,说明这部分能力在项目定位中占比较高,阅读时值得优先关注。",
"importance": "中",
"evidence": "metadata.topics contains: agents"
},
{
"name": "AI Agent 应用",
"category": "Agent 机制",
"description": "仓库主题标签明确指向“AI Agent 应用”方向,说明这部分能力在项目定位中占比较高,阅读时值得优先关注。",
"importance": "中",
"evidence": "metadata.topics contains: ai-agents"
},
{
"name": "生成式 AI 应用",
"category": "大模型应用",
"description": "仓库主题标签明确指向“生成式 AI 应用”方向,说明这部分能力在项目定位中占比较高,阅读时值得优先关注。",
"importance": "中",
"evidence": "metadata.topics contains: generative-ai"
},
{
"name": "LangGraph 编排",
"category": "Agent 编排",
"description": "仓库主题标签明确指向“LangGraph 编排”方向,说明这部分能力在项目定位中占比较高,阅读时值得优先关注。",
"importance": "中",
"evidence": "metadata.topics contains: langgraph"
},
{
"name": "大模型应用",
"category": "大模型应用",
"description": "仓库主题标签明确指向“大模型应用”方向,说明这部分能力在项目定位中占比较高,阅读时值得优先关注。",
"importance": "中",
"evidence": "metadata.topics contains: llm"
},
{
"name": "Prompt 设计",
"category": "LLM 应用框架",
"description": "Prompt 设计 是理解这一类仓库实现思路时几乎都会遇到的核心主题,适合拿来建立整体认知。",
"importance": "中",
"evidence": "inferred from repo_category: LLM 应用框架"
},
{
"name": "检索链路",
"category": "LLM 应用框架",
"description": "检索链路 是理解这一类仓库实现思路时几乎都会遇到的核心主题,适合拿来建立整体认知。",
"importance": "中",
"evidence": "inferred from repo_category: LLM 应用框架"
}
],
"suggested_learning_path": [
"先通读 README,明确项目解决的问题和核心定位",
"再结合目录结构定位入口文件、核心模块和示例代码",
"最后挑 1 到 2 个代表性模块深入阅读实现细节",
"重点关注 Prompt、检索、工具调用或 Agent 相关模块",
"建议优先阅读目录:libs"
],
"relevant_directories": [
"libs"
],
"tags": [
"Python",
"agents",
"ai",
"ai-agents",
"anthropic",
"chatgpt",
"deepagents",
"enterprise"
]
}
\ No newline at end of file
{
"repo_name": "vscode",
"repo_url": "https://github.com/microsoft/vscode",
"repo_category": "开发工具",
"summary": "Visual Studio Code is a distribution of the Code - OSS repository with Microsoft-specific customizations released under a traditional Microsoft product license.",
"learning_value": "vscode 适合作为 开发工具 的学习样本,可以帮助理解 TypeScript 生态下的核心能力、工程组织方式以及真实项目的功能拆分。",
"prerequisites": [
"TypeScript 基础",
"软件工程基础",
"调试与插件机制基础"
],
"knowledge_points": [
{
"name": "编辑器产品定位",
"category": "核心能力",
"description": "README 将这一项作为核心能力直接强调,适合作为理解仓库定位的第一优先点。",
"importance": "高",
"evidence": "README main_features: Visual Studio Code"
},
{
"name": "编辑器架构",
"category": "开发工具",
"description": "仓库主题标签明确指向“编辑器架构”方向,说明这部分能力在项目定位中占比较高,阅读时值得优先关注。",
"importance": "中",
"evidence": "metadata.topics contains: editor"
},
{
"name": "Electron 桌面壳",
"category": "开发工具",
"description": "仓库主题标签明确指向“Electron 桌面壳”方向,说明这部分能力在项目定位中占比较高,阅读时值得优先关注。",
"importance": "中",
"evidence": "metadata.topics contains: electron"
},
{
"name": "VS Code 生态",
"category": "开发工具",
"description": "仓库主题标签明确指向“VS Code 生态”方向,说明这部分能力在项目定位中占比较高,阅读时值得优先关注。",
"importance": "中",
"evidence": "metadata.topics contains: visual-studio-code"
},
{
"name": "阅读 src 目录",
"category": "核心源码",
"description": "src 目录通常承载与 核心源码 相关的关键实现,适合作为进入源码细节时的优先切入点。",
"importance": "中",
"evidence": "directory_tree contains top-level entry: src"
},
{
"name": "阅读 extensions 目录",
"category": "扩展机制",
"description": "extensions 目录通常承载与 扩展机制 相关的关键实现,适合作为进入源码细节时的优先切入点。",
"importance": "中",
"evidence": "directory_tree contains top-level entry: extensions"
},
{
"name": "阅读 scripts 目录",
"category": "工程脚本",
"description": "scripts 目录通常承载与 工程脚本 相关的关键实现,适合作为进入源码细节时的优先切入点。",
"importance": "中",
"evidence": "directory_tree contains top-level entry: scripts"
},
{
"name": "阅读 build 目录",
"category": "构建系统",
"description": "build 目录通常承载与 构建系统 相关的关键实现,适合作为进入源码细节时的优先切入点。",
"importance": "中",
"evidence": "directory_tree contains top-level entry: build"
}
],
"suggested_learning_path": [
"先通读 README,明确项目解决的问题和核心定位",
"再结合目录结构定位入口文件、核心模块和示例代码",
"最后挑 1 到 2 个代表性模块深入阅读实现细节",
"建议优先阅读目录:src, extensions, scripts"
],
"relevant_directories": [
"src",
"extensions",
"scripts",
"build",
"remote",
"cli",
"resources",
"test"
],
"tags": [
"TypeScript",
"editor",
"electron",
"microsoft",
"typescript",
"visual-studio-code"
]
}
\ No newline at end of file
{
"repo_name": "pytorch",
"repo_url": "https://github.com/pytorch/pytorch",
"repo_category": "深度学习框架",
"summary": "PyTorch is a Python package that provides two high-level features: - Tensor computation (like NumPy) with strong GPU acceleration - Deep neural networks built on a tape-based autograd system",
"learning_value": "pytorch 适合作为 深度学习框架 的学习样本,可以帮助理解 Python 生态下的核心能力、工程组织方式以及真实项目的功能拆分。",
"prerequisites": [
"Python 基础",
"线性代数与张量基础",
"深度学习基本概念"
],
"knowledge_points": [
{
"name": "Tensor computation (like NumPy) with strong GPU acceleration",
"category": "核心能力",
"description": "README 将这一项作为核心能力直接强调,适合作为理解仓库定位的第一优先点。",
"importance": "高",
"evidence": "README main_features: Tensor computation (like NumPy) with strong GPU acceleration"
},
{
"name": "Deep neural networks built on a tape-based autograd system",
"category": "核心能力",
"description": "README 将这一项作为核心能力直接强调,适合作为理解仓库定位的第一优先点。",
"importance": "高",
"evidence": "README main_features: Deep neural networks built on a tape-based autograd system"
},
{
"name": "A replacement for NumPy to use the power of GPUs",
"category": "核心能力",
"description": "README 将这一项作为核心能力直接强调,适合作为理解仓库定位的第一优先点。",
"importance": "高",
"evidence": "README main_features: A replacement for NumPy to use the power of GPUs"
},
{
"name": "A deep learning research platform that provides maximum flexibility and speed",
"category": "核心能力",
"description": "README 将这一项作为核心能力直接强调,适合作为理解仓库定位的第一优先点。",
"importance": "高",
"evidence": "README main_features: A deep learning research platform that provides maximum flexibility and speed"
},
{
"name": "autograd",
"category": "深度学习框架",
"description": "仓库主题标签明确指向“autograd”方向,说明这部分能力在项目定位中占比较高,阅读时值得优先关注。",
"importance": "中",
"evidence": "metadata.topics contains: autograd"
},
{
"name": "深度学习",
"category": "深度学习",
"description": "仓库主题标签明确指向“深度学习”方向,说明这部分能力在项目定位中占比较高,阅读时值得优先关注。",
"importance": "中",
"evidence": "metadata.topics contains: deep-learning"
},
{
"name": "gpu",
"category": "深度学习框架",
"description": "仓库主题标签明确指向“gpu”方向,说明这部分能力在项目定位中占比较高,阅读时值得优先关注。",
"importance": "中",
"evidence": "metadata.topics contains: gpu"
},
{
"name": "机器学习",
"category": "机器学习",
"description": "仓库主题标签明确指向“机器学习”方向,说明这部分能力在项目定位中占比较高,阅读时值得优先关注。",
"importance": "中",
"evidence": "metadata.topics contains: machine-learning"
}
],
"suggested_learning_path": [
"先通读 README,明确项目解决的问题和核心定位",
"再结合目录结构定位入口文件、核心模块和示例代码",
"最后挑 1 到 2 个代表性模块深入阅读实现细节",
"重点关注 Tensor、自动求导、模型组件和训练相关模块",
"建议优先阅读目录:docs, scripts, android"
],
"relevant_directories": [
"docs",
"scripts",
"android",
"aten",
"benchmarks",
"binaries",
"c10",
"caffe2"
],
"tags": [
"Python",
"autograd",
"deep-learning",
"gpu",
"machine-learning",
"neural-network",
"numpy",
"python"
]
}
\ No newline at end of file
{
"repo_name": "llama_index",
"repo_url": "https://github.com/run-llama/llama_index",
"repo_category": "LLM 应用框架",
"summary": "LlamaIndex OSS (by LlamaIndex) is an open-source framework to build agentic applications. **Parse** is our enterprise platform for agentic OCR, parsing, extraction, indexing and more. You can use LlamaParse with this framework or on its own; see LlamaParse below for signup and product links.",
"learning_value": "llama_index 适合作为 LLM 应用框架 的学习样本,可以帮助理解 Python 生态下的核心能力、工程组织方式以及真实项目的功能拆分。",
"prerequisites": [
"Python 基础",
"Prompt 基础",
"向量检索或 RAG 基础"
],
"knowledge_points": [
{
"name": "🚀 Overview",
"category": "核心能力",
"description": "README 将这一项作为核心能力直接强调,适合作为理解仓库定位的第一优先点。",
"importance": "高",
"evidence": "README main_features: 🚀 Overview"
},
{
"name": "💡 Contributing",
"category": "核心能力",
"description": "README 将这一项作为核心能力直接强调,适合作为理解仓库定位的第一优先点。",
"importance": "高",
"evidence": "README main_features: 💡 Contributing"
},
{
"name": "📄 Documentation",
"category": "核心能力",
"description": "README 将这一项作为核心能力直接强调,适合作为理解仓库定位的第一优先点。",
"importance": "高",
"evidence": "README main_features: 📄 Documentation"
},
{
"name": "💻 Example Usage",
"category": "核心能力",
"description": "README 将这一项作为核心能力直接强调,适合作为理解仓库定位的第一优先点。",
"importance": "高",
"evidence": "README main_features: 💻 Example Usage"
},
{
"name": "A note on Verification of Build Assets",
"category": "核心能力",
"description": "README 将这一项作为核心能力直接强调,适合作为理解仓库定位的第一优先点。",
"importance": "高",
"evidence": "README main_features: A note on Verification of Build Assets"
},
{
"name": "📖 Citation",
"category": "核心能力",
"description": "README 将这一项作为核心能力直接强调,适合作为理解仓库定位的第一优先点。",
"importance": "高",
"evidence": "README main_features: 📖 Citation"
},
{
"name": "Agent 机制",
"category": "Agent 机制",
"description": "仓库主题标签明确指向“Agent 机制”方向,说明这部分能力在项目定位中占比较高,阅读时值得优先关注。",
"importance": "中",
"evidence": "metadata.topics contains: agents"
},
{
"name": "application",
"category": "LLM 应用框架",
"description": "仓库主题标签明确指向“application”方向,说明这部分能力在项目定位中占比较高,阅读时值得优先关注。",
"importance": "中",
"evidence": "metadata.topics contains: application"
}
],
"suggested_learning_path": [
"先通读 README,明确项目解决的问题和核心定位",
"再结合目录结构定位入口文件、核心模块和示例代码",
"最后挑 1 到 2 个代表性模块深入阅读实现细节",
"重点关注 Prompt、检索、工具调用或 Agent 相关模块",
"建议优先阅读目录:docs, scripts, llama-dev"
],
"relevant_directories": [
"docs",
"scripts",
"llama-dev",
"llama-index-core",
"llama-index-instrumentation",
"llama-index-integrations",
"llama-index-utils"
],
"tags": [
"Python",
"agents",
"application",
"data",
"fine-tuning",
"framework",
"llamaindex",
"llm"
]
}
\ No newline at end of file
{
"repo_name": "scikit-learn",
"repo_url": "https://github.com/scikit-learn/scikit-learn",
"repo_category": "机器学习库",
"summary": "**scikit-learn** is a Python module for machine learning built on top of SciPy and is distributed under the 3-Clause BSD license.",
"learning_value": "scikit-learn 适合作为 机器学习库 的学习样本,可以帮助理解 Python 生态下的核心能力、工程组织方式以及真实项目的功能拆分。",
"prerequisites": [
"Python 基础",
"机器学习基本概念",
"数据预处理与特征工程基础"
],
"knowledge_points": [
{
"name": "Data analysis",
"category": "核心能力",
"description": "README 将这一项作为核心能力直接强调,适合作为理解仓库定位的第一优先点。",
"importance": "高",
"evidence": "README main_features: Data analysis"
},
{
"name": "Data science",
"category": "核心能力",
"description": "README 将这一项作为核心能力直接强调,适合作为理解仓库定位的第一优先点。",
"importance": "高",
"evidence": "README main_features: Data science"
},
{
"name": "机器学习能力",
"category": "核心能力",
"description": "README 将这一项作为核心能力直接强调,适合作为理解仓库定位的第一优先点。",
"importance": "高",
"evidence": "README main_features: Machine learning"
},
{
"name": "统计方法基础",
"category": "核心能力",
"description": "README 将这一项作为核心能力直接强调,适合作为理解仓库定位的第一优先点。",
"importance": "高",
"evidence": "README main_features: Statistics"
},
{
"name": "数据分析",
"category": "机器学习库",
"description": "仓库主题标签明确指向“数据分析”方向,说明这部分能力在项目定位中占比较高,阅读时值得优先关注。",
"importance": "中",
"evidence": "metadata.topics contains: data-analysis"
},
{
"name": "数据科学",
"category": "机器学习库",
"description": "仓库主题标签明确指向“数据科学”方向,说明这部分能力在项目定位中占比较高,阅读时值得优先关注。",
"importance": "中",
"evidence": "metadata.topics contains: data-science"
},
{
"name": "机器学习",
"category": "机器学习",
"description": "仓库主题标签明确指向“机器学习”方向,说明这部分能力在项目定位中占比较高,阅读时值得优先关注。",
"importance": "中",
"evidence": "metadata.topics contains: machine-learning"
},
{
"name": "统计建模",
"category": "机器学习库",
"description": "仓库主题标签明确指向“统计建模”方向,说明这部分能力在项目定位中占比较高,阅读时值得优先关注。",
"importance": "中",
"evidence": "metadata.topics contains: statistics"
}
],
"suggested_learning_path": [
"先通读 README,明确项目解决的问题和核心定位",
"再结合目录结构定位入口文件、核心模块和示例代码",
"最后挑 1 到 2 个代表性模块深入阅读实现细节",
"重点关注数据预处理、模型接口、训练评估和算法模块",
"建议优先阅读目录:sklearn, examples, asv_benchmarks"
],
"relevant_directories": [
"sklearn",
"examples",
"asv_benchmarks",
"benchmarks",
"doc",
"build_tools",
"maint_tools"
],
"tags": [
"Python",
"data-analysis",
"data-science",
"machine-learning",
"python",
"statistics"
]
}
\ No newline at end of file
{
"repo_name": "streamlit",
"repo_url": "https://github.com/streamlit/streamlit",
"repo_category": "数据应用框架",
"summary": "Streamlit lets you transform Python scripts into interactive web apps in minutes, instead of weeks. Build dashboards, generate reports, or create chat apps. Once you’ve created an app, you can use our Community Cloud platform to deploy, manage, and share your app.",
"learning_value": "streamlit 适合作为 数据应用框架 的学习样本,可以帮助理解 Python 生态下的核心能力、工程组织方式以及真实项目的功能拆分。",
"prerequisites": [
"Python 基础",
"Python 数据处理基础",
"Web 交互与可视化基础"
],
"knowledge_points": [
{
"name": "Pythonic 开发体验",
"category": "核心能力",
"description": "仓库强调这一能力,重点在于:Write beautiful, easy-to-read code",
"importance": "高",
"evidence": "README main_features: Simple and Pythonic: Write beautiful, easy-to-read code"
},
{
"name": "Fast, interactive prototyping",
"category": "核心能力",
"description": "仓库强调这一能力,重点在于:Let others interact with your data and provide feedback quickly",
"importance": "高",
"evidence": "README main_features: Fast, interactive prototyping: Let others interact with your data and provide feedback quickly"
},
{
"name": "实时编辑反馈",
"category": "核心能力",
"description": "仓库强调这一能力,重点在于:See your app update instantly as you edit your script",
"importance": "高",
"evidence": "README main_features: Live editing: See your app update instantly as you edit your script"
},
{
"name": "开源与社区生态",
"category": "核心能力",
"description": "仓库强调这一能力,重点在于:Join a vibrant community and contribute to Streamlit's future",
"importance": "高",
"evidence": "README main_features: Open-source and free: Join a vibrant community and contribute to Streamlit's future"
},
{
"name": "数据分析",
"category": "数据应用框架",
"description": "仓库主题标签明确指向“数据分析”方向,说明这部分能力在项目定位中占比较高,阅读时值得优先关注。",
"importance": "中",
"evidence": "metadata.topics contains: data-analysis"
},
{
"name": "数据科学",
"category": "数据应用框架",
"description": "仓库主题标签明确指向“数据科学”方向,说明这部分能力在项目定位中占比较高,阅读时值得优先关注。",
"importance": "中",
"evidence": "metadata.topics contains: data-science"
},
{
"name": "数据可视化",
"category": "数据应用框架",
"description": "仓库主题标签明确指向“数据可视化”方向,说明这部分能力在项目定位中占比较高,阅读时值得优先关注。",
"importance": "中",
"evidence": "metadata.topics contains: data-visualization"
},
{
"name": "深度学习",
"category": "深度学习",
"description": "仓库主题标签明确指向“深度学习”方向,说明这部分能力在项目定位中占比较高,阅读时值得优先关注。",
"importance": "中",
"evidence": "metadata.topics contains: deep-learning"
}
],
"suggested_learning_path": [
"先通读 README,明确项目解决的问题和核心定位",
"再结合目录结构定位入口文件、核心模块和示例代码",
"最后挑 1 到 2 个代表性模块深入阅读实现细节",
"重点关注页面组件、数据展示、交互逻辑和部署相关模块",
"建议优先阅读目录:frontend, lib, scripts"
],
"relevant_directories": [
"frontend",
"lib",
"scripts",
"proto",
"e2e_playwright",
"specs",
"wiki"
],
"tags": [
"Python",
"data-analysis",
"data-science",
"data-visualization",
"deep-learning",
"developer-tools",
"machine-learning",
"python"
]
}
\ No newline at end of file
{
"repo_name": "vllm",
"repo_url": "https://github.com/vllm-project/vllm",
"repo_category": "LLM 应用框架",
"summary": "vLLM is a fast and easy-to-use library for LLM inference and serving.",
"learning_value": "vllm 适合作为 LLM 应用框架 的学习样本,可以帮助理解 Python 生态下的核心能力、工程组织方式以及真实项目的功能拆分。",
"prerequisites": [
"Python 基础",
"Prompt 基础",
"向量检索或 RAG 基础"
],
"knowledge_points": [
{
"name": "Contact Us",
"category": "核心能力",
"description": "README 将这一项作为核心能力直接强调,适合作为理解仓库定位的第一优先点。",
"importance": "高",
"evidence": "README main_features: Contact Us"
},
{
"name": "Media Kit",
"category": "核心能力",
"description": "README 将这一项作为核心能力直接强调,适合作为理解仓库定位的第一优先点。",
"importance": "高",
"evidence": "README main_features: Media Kit"
},
{
"name": "amd",
"category": "LLM 应用框架",
"description": "仓库主题标签明确指向“amd”方向,说明这部分能力在项目定位中占比较高,阅读时值得优先关注。",
"importance": "中",
"evidence": "metadata.topics contains: amd"
},
{
"name": "blackwell",
"category": "LLM 应用框架",
"description": "仓库主题标签明确指向“blackwell”方向,说明这部分能力在项目定位中占比较高,阅读时值得优先关注。",
"importance": "中",
"evidence": "metadata.topics contains: blackwell"
},
{
"name": "cuda",
"category": "LLM 应用框架",
"description": "仓库主题标签明确指向“cuda”方向,说明这部分能力在项目定位中占比较高,阅读时值得优先关注。",
"importance": "中",
"evidence": "metadata.topics contains: cuda"
},
{
"name": "Deepseek V3",
"category": "LLM 应用框架",
"description": "仓库主题标签明确指向“Deepseek V3”方向,说明这部分能力在项目定位中占比较高,阅读时值得优先关注。",
"importance": "中",
"evidence": "metadata.topics contains: deepseek-v3"
},
{
"name": "gpt",
"category": "LLM 应用框架",
"description": "仓库主题标签明确指向“gpt”方向,说明这部分能力在项目定位中占比较高,阅读时值得优先关注。",
"importance": "中",
"evidence": "metadata.topics contains: gpt"
},
{
"name": "阅读 docs 目录",
"category": "文档体系",
"description": "docs 目录通常承载与 文档体系 相关的关键实现,适合作为进入源码细节时的优先切入点。",
"importance": "中",
"evidence": "directory_tree contains top-level entry: docs"
}
],
"suggested_learning_path": [
"先通读 README,明确项目解决的问题和核心定位",
"再结合目录结构定位入口文件、核心模块和示例代码",
"最后挑 1 到 2 个代表性模块深入阅读实现细节",
"重点关注 Prompt、检索、工具调用或 Agent 相关模块",
"建议优先阅读目录:vllm, docs, examples"
],
"relevant_directories": [
"vllm",
"docs",
"examples",
"tests",
"scripts",
"benchmarks",
"cmake",
"csrc"
],
"tags": [
"Python",
"amd",
"blackwell",
"cuda",
"deepseek",
"deepseek-v3",
"gpt",
"gpt-oss"
]
}
\ No newline at end of file
{
"title": "repo_knowledge_extraction",
"type": "object",
"additionalProperties": false,
"required": [
"repo_name",
"repo_url",
"repo_category",
"summary",
"learning_value",
"prerequisites",
"knowledge_points",
"suggested_learning_path",
"relevant_directories",
"tags"
],
"properties": {
"repo_name": {
"type": "string"
},
"repo_url": {
"type": "string"
},
"repo_category": {
"type": "string"
},
"summary": {
"type": "string"
},
"learning_value": {
"type": "string"
},
"prerequisites": {
"type": "array",
"items": {
"type": "string"
}
},
"knowledge_points": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"required": [
"name",
"category",
"description",
"importance",
"evidence"
],
"properties": {
"name": {
"type": "string"
},
"category": {
"type": "string"
},
"description": {
"type": "string"
},
"importance": {
"type": "string"
},
"evidence": {
"type": "string"
}
}
}
},
"suggested_learning_path": {
"type": "array",
"items": {
"type": "string"
}
},
"relevant_directories": {
"type": "array",
"items": {
"type": "string"
}
},
"tags": {
"type": "array",
"items": {
"type": "string"
}
}
}
}
\ No newline at end of file
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑!
请 注册 或者 后发表评论