Gitbook

SimpleGitbookReader

Bases: BaseReader

简单的 Gitbook 阅读器。

将每个 Gitbook 页面转换为 LlamaIndex 使用的 Document。

参数

名称	类型	描述	默认值
api_token	str	Gitbook API 令牌。	必需
api_url	str	Gitbook API 端点。	无

源代码位于 llama-index-integrations/readers/llama-index-readers-gitbook/llama_index/readers/gitbook/base.py

class SimpleGitbookReader(BaseReader):
    """
    Simple gitbook reader.

    Convert each gitbook page into Document used by LlamaIndex.

    Args:
        api_token (str): Gitbook API Token.
        api_url (str): Gitbook API Endpoint.

    """

    def __init__(self, api_token: str, api_url: str = None) -> None:
        """Initialize with parameters."""
        self.client = GitbookClient(api_token, api_url)

    def load_data(
        self,
        space_id: str,
        metadata_names: Optional[List[str]] = None,
        show_progress=False,
    ) -> List[Document]:
        """
        Load data from the input directory.

        Args:
            space_id (str): Gitbook space id
            metadata_names (Optional[List[str]]): names of the fields to be added
                to the metadata attribute of the Document.
                only 'path', 'title', 'description', 'parent' are available
                Defaults to None
            show_progress (bool, optional): Show progress bar. Defaults to False

        Returns:
            List[Document]: A list of documents.

        """
        if metadata_names:
            invalid_fields = set(metadata_names) - VALID_METADATA_FIELDS
            if invalid_fields:
                raise ValueError(
                    f"Invalid metadata fields: {', '.join(invalid_fields)}"
                )

        documents = []
        pages = self.client.list_pages(space_id)

        if show_progress:
            from tqdm import tqdm

            iterator = tqdm(pages, desc="Downloading pages")
        else:
            iterator = pages

        for page in iterator:
            id = page.get("id")
            content = self.client.get_page_markdown(space_id, id)
            if not content:
                print(f"Warning: No content found for page ID {id}. Skipping...")
                continue

            if metadata_names is None:
                documents.append(
                    Document(text=content, id_=id, metadata={"path": page.get("path")})
                )
            else:
                try:
                    metadata = {name: page.get(name) for name in metadata_names}
                except KeyError as err:
                    raise ValueError(
                        f"{err.args[0]} field is not available. Choose from {', '.join(VALID_METADATA_FIELDS)}"
                    ) from err
                documents.append(Document(text=content, id_=id, metadata=metadata))

        return documents

load_data

load_data(space_id: str, metadata_names: Optional[List[str]] = None, show_progress=False) -> List[Document]

从输入目录加载数据。

参数

名称	类型	描述	默认值
space_id	str	Gitbook 空间 ID	必需
metadata_names	Optional[List[str]]	要添加到 Document 元数据属性的字段名称。仅 'path', 'title', 'description', 'parent' 可用。默认为 None	无
show_progress	bool	显示进度条。默认为 False	False

返回值

类型	描述
List[Document]	List[Document]: 文档列表。

源代码位于 llama-index-integrations/readers/llama-index-readers-gitbook/llama_index/readers/gitbook/base.py

def load_data(
    self,
    space_id: str,
    metadata_names: Optional[List[str]] = None,
    show_progress=False,
) -> List[Document]:
    """
    Load data from the input directory.

    Args:
        space_id (str): Gitbook space id
        metadata_names (Optional[List[str]]): names of the fields to be added
            to the metadata attribute of the Document.
            only 'path', 'title', 'description', 'parent' are available
            Defaults to None
        show_progress (bool, optional): Show progress bar. Defaults to False

    Returns:
        List[Document]: A list of documents.

    """
    if metadata_names:
        invalid_fields = set(metadata_names) - VALID_METADATA_FIELDS
        if invalid_fields:
            raise ValueError(
                f"Invalid metadata fields: {', '.join(invalid_fields)}"
            )

    documents = []
    pages = self.client.list_pages(space_id)

    if show_progress:
        from tqdm import tqdm

        iterator = tqdm(pages, desc="Downloading pages")
    else:
        iterator = pages

    for page in iterator:
        id = page.get("id")
        content = self.client.get_page_markdown(space_id, id)
        if not content:
            print(f"Warning: No content found for page ID {id}. Skipping...")
            continue

        if metadata_names is None:
            documents.append(
                Document(text=content, id_=id, metadata={"path": page.get("path")})
            )
        else:
            try:
                metadata = {name: page.get(name) for name in metadata_names}
            except KeyError as err:
                raise ValueError(
                    f"{err.args[0]} field is not available. Choose from {', '.join(VALID_METADATA_FIELDS)}"
                ) from err
            documents.append(Document(text=content, id_=id, metadata=metadata))

    return documents