改めてLlamaIndexを試してみる

$ pyenv virtualenv 3.10.13 llamaindex
$ mkdir llamaindex && cd llamaindex
$ pyenv local llamaindex

$ pip install jupyterlab ipywidgets

$ echo "OPENAI_API_KEY=XXXXXXXXXXXXXXXXX" > .env

$ jupyter-lab --ip='0.0.0.0' --NotebookApp.token=''

!pip install llama-index python-dotenv

from dotenv import load_dotenv

load_dotenv()

%%bash

wget https://raw.githubusercontent.com/run-llama/llama_index/main/examples/paul_graham_essay/data/paul_graham_essay.txt
mkdir data
mv paul_graham_essay.txt data

from llama_index import VectorStoreIndex, SimpleDirectoryReader

documents = SimpleDirectoryReader('data').load_data()
index = VectorStoreIndex.from_documents(documents)

query_engine = index.as_query_engine()
response = query_engine.query("What did the author do growing up?")
print(response)

The author worked on writing and programming outside of school before college. They wrote short stories and tried writing programs on an IBM 1401 computer using an early version of Fortran. They later got a microcomputer and started programming on it, writing simple games and a word processor. They also mentioned their interest in philosophy and AI.

import logging
import sys

logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))

openai.log = "debug"

openai.log = "None"

index.storage_context.persist()

$ tree storage/
storage/
├── docstore.json
├── graph_store.json
├── index_store.json
└── vector_store.json

from llama_index import StorageContext, load_index_from_storage

storage_context = StorageContext.from_defaults(persist_dir="./storage")
index = load_index_from_storage(storage_context)

from llama_index import SimpleDirectoryReader

documents = SimpleDirectoryReader('./data').load_data()

print(len(documents))
print(documents)

1
[
  Document(
    id_='0f62af38-49aa-4935-b9e3-beb965597f85',
    embedding=None,
    metadata={},
    excluded_embed_metadata_keys=[],
    excluded_llm_metadata_keys=[],
    relationships={},
    hash='2e2d9629223c077019a6dde689049344ff2293d6c52372871420119ec049f25c',
    text='\n\nWhat I Worked On\n\nFebruary 2021\n\nBefore college the two main things I worked on, outside of school, were writing and programming. I didn\'t write essays. (snip) So it seems likely there exists at least one path out of McCarthy\'s Lisp along which discoveredness is preserved.\n\n\n\nThanks to Trevor Blackwell, John Collison, Patrick Collison, Daniel Gackle, Ralph Hazell, Jessica Livingston, Robert Morris, and Harj Taggar for reading drafts of this.\n','
    start_char_idx=None,
    end_char_idx=None,
    text_template='{metadata_str}\n\n{content}', 
    metadata_template='{key}: {value}',
    metadata_seperator='\n')
]

import glob
import os
from llama_index import Document

text_list = []

for file in glob.glob('data/*'):
    with open(file, encoding='utf-8') as f:
        c = f.read()
        text_list.append(c)

documents = [Document(text=t) for t in text_list]

from llama_index.node_parser import SimpleNodeParser

parser = SimpleNodeParser.from_defaults()

nodes = parser.get_nodes_from_documents(documents)

print(len(nodes))
print(nodes)

18
[
  TextNode(
    id_='55bf4b17-053b-4a6d-bdb2-ed24e7b2a782',
    embedding=None,
    metadata={},
    excluded_embed_metadata_keys=[],
    excluded_llm_metadata_keys=[],
    relationships={
        <NodeRelationship.SOURCE: '1'>: RelatedNodeInfo(
          node_id='fe4bbbd0-df5c-4ed7-bdff-27c04e60064d',
          node_type=None,
          metadata={},
          hash='2e2d9629223c077019a6dde689049344ff2293d6c52372871420119ec049f25c'
        ),
        <NodeRelationship.NEXT: '3'>: RelatedNodeInfo(
          node_id='ee57542d-ee5c-41ec-80c9-67f07a3568df',
          node_type=None,
          metadata={},
          hash='2b076feb45ef9318654997e47fb3b2f2185bcc7e39e0997045fcf755cb2ca6dc'
        )
    },
    hash='004bbb820e2627b96db9a79d93bb1b8a665582ada07e261a6085d2d0b0119ad6',
    text='What I Worked On\n\nFebruary 2021\n\nBefore college the two main things I worked on, outside of school, were writing and programming. I didn\'t write essays. (snip) I haven\'t tried rereading The Moon is a Harsh Mistress, so I don\'t know how well it has aged, but when I read it I was drawn entirely into its world. It seemed only a matter of time before we\'d have Mike, and when I saw Winograd using SHRDLU, it seemed like that time would be a few years at most. All you had to do was teach SHRDLU more words.',
    start_char_idx=None,
    end_char_idx=None,
    text_template='{metadata_str}\n\n{content}',
    metadata_template='{key}: {value}',
    metadata_seperator='\n'
  ),
  TextNode(
    id_='ee57542d-ee5c-41ec-80c9-67f07a3568df',
    (snip)
    relationships={
      <NodeRelationship.SOURCE: '1'>: RelatedNodeInfo(
        node_id='fe4bbbd0-df5c-4ed7-bdff-27c04e60064d',
        node_type=None,
        metadata={}, 
        hash='2e2d9629223c077019a6dde689049344ff2293d6c52372871420119ec049f25c'
      ),
      <NodeRelationship.PREVIOUS: '2'>: RelatedNodeInfo(
        node_id='55bf4b17-053b-4a6d-bdb2-ed24e7b2a782',
        node_type=None,
        metadata={}, 
        hash='004bbb820e2627b96db9a79d93bb1b8a665582ada07e261a6085d2d0b0119ad6'
      ),
      (snip)
]

rom llama_index.schema import TextNode, NodeRelationship, RelatedNodeInfo

node1 = TextNode(text="<text_chunk>", id_="<node_id>")
node2 = TextNode(text="<text_chunk>", id_="<node_id>")
# set relationships
node1.relationships[NodeRelationship.NEXT] = RelatedNodeInfo(node_id=node2.node_id)
node2.relationships[NodeRelationship.PREVIOUS] = RelatedNodeInfo(node_id=node1.node_id)
nodes = [node1, node2]

node2.relationships[NodeRelationship.PARENT] = RelatedNodeInfo(node_id=node1.node_id, metadata={"key": "val"})

node_parser = SimpleNodeParser.from_defaults(chunk_size=1024, chunk_overlap=20)

import tiktoken
from llama_index.text_splitter import SentenceSplitter

text_splitter = SentenceSplitter(
  separator=" ",
  chunk_size=1024,
  chunk_overlap=20,
  paragraph_separator="\n\n\n",
  secondary_chunking_regex="[^,.;。]+[,.;。]?",
  tokenizer=tiktoken.encoding_for_model("gpt-3.5-turbo").encode
)

node_parser = SimpleNodeParser.from_defaults(text_splitter=text_splitter)

from llama_index import VectorStoreIndex

index = VectorStoreIndex.from_documents(documents)

from llama_index import VectorStoreIndex

index = VectorStoreIndex(nodes)

from llama_index import StorageContext
from llama_index import VectorStoreIndex, SummaryIndex

storage_context = StorageContext.from_defaults()
storage_context.docstore.add_documents(nodes)

index1 = VectorStoreIndex(nodes, storage_context=storage_context)
index2 = SummaryIndex(nodes, storage_context=storage_context)

from llama_index import VectorStoreIndex

index = VectorStoreIndex([])
for doc in documents:
    index.insert(doc)

from llama_index import VectorStoreIndex

index = VectorStoreIndex([])
index.insert_nodes(nodes)

document = Document(
    text='text',
    metadata={
        'filename': '<doc_file_name>',
        'category': '<category>'
    }
)

from llama_index import VectorStoreIndex, ServiceContext, set_global_service_context
from llama_index.llms import OpenAI

...

llm = OpenAI(model="gpt-4", temperature=0, max_tokens=256)

service_context = ServiceContext.from_defaults(llm=llm)
set_global_service_context(service_context)

index = VectorStoreIndex.from_documents(
    documents
)

from llama_index import VectorStoreIndex, ServiceContext, set_global_service_context
from llama_index.llms import OpenAI
from llama_index.embeddings import OpenAIEmbedding

embed_model = OpenAIEmbedding()
llm = OpenAI(temperature=0.1, model="gpt-4")

service_context = ServiceContext.from_defaults(
    embed_model=embed_model,
    llm=llm
)
set_global_service_context(service_context)

from llama_index import ServiceContext
service_context = ServiceContext.from_defaults(llm="local")