Migliorare il RAG:

   OPENAI_API_KEY=sk-......

   import dotenv
   from ChatDocument import ChatDocument

   def main():
       dotenv.load_dotenv(".env.openai")
       sample_text_path = "C:\\Users\\claud\\Downloads\\The Civilisation of the Renaissance in Italy - Jacob Burckhardt.pdf"
       chat_doc = ChatDocument(document_path=sample_text_path)
       chat_doc.ingest(max_chunks=300, max_pages=30)

       while True:
           query = input("Please enter your question (or press Enter to exit): ")
           if not query.strip():
               print("Exiting...")
               break
           response = chat_doc.ask(query)
           print("\nResponse:\n", response, "\n")

   if __name__ == "__main__":
       main()

   import os
   from langchain.chains import RetrievalQA
   from langchain_openai import OpenAIEmbeddings, ChatOpenAI
   from langchain_text_splitters import RecursiveCharacterTextSplitter
   from loguru import logger
   from document_loader import DocumentLoader
   from vector_store import VectorStoreManager

   class ChatDocument:
       def __init__(self, document_path, persist_directory=None):
           self.model = ChatOpenAI(model_name="gpt-4o-mini", openai_api_key=os.getenv("OPENAI_API_KEY"))
           self.document_path = document_path
           self.embedder = OpenAIEmbeddings(openai_api_key=os.getenv("OPENAI_API_KEY"))
           self.text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=150)
           self.persist_directory = persist_directory
           self.vector_store_manager = VectorStoreManager(self.embedder, self.persist_directory)
           logger.info("ChatDocument initialized")

       def ingest(self, batch_size=10, max_chunks=10, max_pages=9999):
           logger.info(f"Starting ingestion of file: {self.document_path}")
           loader = DocumentLoader(self.document_path).get_loader()
           docs = loader.load(max_pages)
           logger.info(f"Loaded {len(docs)} document(s) from {self.document_path}")

           chunks = self.text_splitter.split_documents(docs)
           logger.info(f"Number of chunks created: {len(chunks)}")

           self.vector_store = self.vector_store_manager.create_vector_store(chunks,  self.document_path, batch_size, max_chunks)
           if not self.vector_store:
               return

           self.retriever = self.vector_store.as_retriever()
           self.qa_chain = RetrievalQA.from_chain_type(llm=self.model, chain_type="stuff", retriever=self.retriever, return_source_documents=True)
           logger.info("QA chain configured successfully")

       def ask(self, query: str):
           if not self.qa_chain:
               logger.warning("No chain available. Please, add a document first.")
               return "Please, add a document first."
           logger.info(f"Processing query: {query}")

           try:
               response = self.qa_chain.invoke({"query": query})

               formatted_response = f"**Domanda:**\nQ: {query}\n\n"
               formatted_response += f"**Risposta:**\nA: {response['result']}\n\n"
               formatted_response += "**Fonti:**\n"
               for i, doc in enumerate(response['source_documents'], 1):
                   snippet = doc.page_content[:200].replace("\n", " ").strip() + "..."
                   formatted_response += f"{i}. **{doc.metadata['source']}**\n   - \"{snippet}\"\n"

               logger.info(f"Response received: {formatted_response}")
               return formatted_response
           except Exception as e:
               logger.error(f"Error during query processing: {e}")
               return "An error occurred while processing the query."

       def clear(self):
           self.vector_store = None
           self.retriever = None
           self.qa_chain = None
           logger.info("ChatDocument cleared")

   import os
   from langchain_community.document_loaders import TextLoader
   from minichat.RAG_with_gpt_4o_mini.VerbosePyPDFLoader import VerbosePyPDFLoader
   from loguru import logger

   class DocumentLoader:
       def __init__(self, document_path):
           self.document_path = document_path

       def get_loader(self):
           file_path = self.document_path
           ext = os.path.splitext(file_path)[1].lower()
           logger.debug(f"Loading document with extension: {ext}")
           if ext == ".pdf":
               return VerbosePyPDFLoader(file_path)
           elif ext == ".txt":
               return TextLoader(file_path)
           else:
               raise ValueError(f"Unsupported file type: {ext}")

   import os
   import uuid
   from langchain_chroma import Chroma
   from langchain.chains import LLMChain
   from langchain.prompts import PromptTemplate
   from loguru import logger
   from langchain_openai import ChatOpenAI

   class VectorStoreManager:
       def __init__(self, embedder, persist_directory=None, model_name="gpt-4o-mini"):
           self.embedder = embedder
           self.persist_directory = persist_directory or "./chroma_db"
           self.vector_store = None
           self.model = ChatOpenAI(model_name=model_name, openai_api_key=os.getenv("OPENAI_API_KEY"))
           self.summaries = []

       def _summarize_text(self, text):
           prompt_template = PromptTemplate(input_variables=["text"], template="Please provide a concise summary of the following text:\n\n{text}")
           sequence = prompt_template | self.model
           ai_message = sequence.invoke({"text": text})
           summary = ai_message.content.strip()
           return summary

       def create_vector_store(self, chunks, source, batch_size=10, max_chunks=10):
           collection_name = str(uuid.uuid4())
           if max_chunks:
               chunks = chunks[:max_chunks]

           logger.info(f"Creating vector store with {len(chunks)} chunks")

           self.vector_store = Chroma(embedding_function=self.embedder, collection_name=collection_name, persist_directory=self.persist_directory)

           try:
               overall_text = ""
               total_bytes_summarized = 0
               for i in range(0, len(chunks), batch_size):
                   batch_chunks = chunks[i:i + batch_size]
                   texts = [doc.page_content for doc in batch_chunks]
                   metadatas = [{**doc.metadata, 'source': source} for doc in batch_chunks]
                   batch_text = " ".join(texts)
                   batch_bytes = sum(len(text.encode('utf-8')) for text in texts)
                   total_bytes_summarized += batch_bytes

                   batch_summary = self._summarize_text(batch_text)
                   logger.info(f"Batch {i//batch_size + 1} summary: {batch_summary[:80]}... (summarized {batch_bytes} bytes)")

                   self.vector_store.add_texts(texts=texts, metadatas=metadatas)
                   self.vector_store.add_texts(texts=[batch_summary], metadatas=[{'source': source, 'summary': True

, 'batch_number': i//batch_size + 1}])

                   overall_text += batch_text

               overall_summary = self._summarize_text(overall_text)
               overall_summary_bytes = len(overall_text.encode('utf-8'))
               logger.info(f"Overall summary: {overall_summary[:80]}... (summarized {overall_summary_bytes} bytes)")

               self.vector_store.add_texts(texts=[overall_summary], metadatas=[{'source': source, 'summary': True, 'overall': True}])

               logger.info(f"Vector store created and persisted successfully. Total bytes summarized: {total_bytes_summarized}")
           except Exception as e:
               logger.error(f"Failed to create and persist vector store: {e}")
               return None

           return self.vector_store

   langchain==0.2.13
   langchain-openai==0.1.21
   langchain-chroma==0.1.2
   langchain-core==0.2.30
   langchain-community==0.2.11
   loguru==0.7.2
   python-dotenv==1.0.1
   pypdf==4.3.1
   pdf2image==1.17.0
   pytesseract==0.3.10
   Pillow==10.4.0
   chromadb==0.5.5

Question:
Q: fammi un riassunto del testo…

Answer:
A: Il testo analizza vari temi legati al periodo del Rinascimento in Italia, focalizzandosi sull'evoluzione della fama, dell'ingegno, dell'umanesimo e dei cambiamenti sociali. Vengono discussi i punti di vista di Dante sulla fama, l'impatto dell'umanesimo e il revival dell'antichità attraverso la letteratura e l'educazione, con riferimento a figure note come Petrarca e Boccaccio. Inoltre, si esplora la scoperta del mondo naturale e dell'umanità, le scienze naturali e la rappresentazione dell'esperienza umana nella poesia e nella biografia.

Il testo analizza anche le dinamiche religiose e politiche del tempo, evidenziando il malcontento verso le gerarchie monastiche e l'emergere degli ordini mendicanti, insieme all'influenza di Girolamo Savonarola. Si riflette sulla trasformazione delle strutture politiche in Italia, in particolare sotto il regno di Federico II, che centralizzò l'autorità e smantellò i sistemi feudali.

Infine, viene trattata la caduta di un sovrano, il cui regno si frammentò dopo la sua morte, e la violenza dei suoi figli, Giovanni Maria e Filippo Maria, con particolare attenzione alle azioni brutali di Giovanni Maria e alla sua assassinio da parte di cospiratori.

Sources:
1. C:\Users\claud\Downloads\The Civilisation of the Renaissance in Italy - Jacob Burckhardt.pdf
   - "The text outlines various themes related to the Renaissance period, focusing on the evolution of fame, wit, humanism, and societal changes in Italy. Key topics include Dante's views on fame, the impac..."
2. C:\Users\claud\Downloads\The Civilisation of the Renaissance in Italy - Jacob Burckhardt.pdf
   - "The text outlines a historical analysis of religious and political dynamics during the Renaissance, focusing on various themes such as the discontent with monastic hierarchies, the emergence of mendic..."
3. C:\Users\claud\Downloads\The Civilisation of the Renaissance in Italy - Jacob Burckhardt.pdf
   - "The text discusses the downfall of a ruler whose territories, after his death, fragmented despite the significant financial contributions he received during his reign. His sons, Giovanni Maria and Fil..."
4. C:\Users\claud\Downloads\The Civilisation of the Renaissance in Italy - Jacob Burckhardt.pdf
   - "THE CIVILISATION OF THE RENAISSANCE IN FTALY His DS ACR AAR ET ALD THORISEDR TRASSLATIOS HY Si. MDL ACHE..."