From c4c70d6fdc9221aec4216250d5d89a0c00d7d1b6 Mon Sep 17 00:00:00 2001 From: Daniel Soares Martins Date: Mon, 20 Jan 2025 19:32:11 -0300 Subject: [PATCH 1/4] Remove bibliotecas.list and update requirements.txt; add prompt.py and bibliotecas.py for enhanced library tracking and AI integration --- README.md | 38 ++++++++- bibliotecas.list | 13 --- bibliotecas.py | 142 ++++++++++++++++++++++++++++++++ cacheVariables.py | 2 +- getNews.py | 204 ++++++++++++++++++++++++++++++++-------------- prompt.py | 135 ++++++++++++++++++++++++++++++ requirements.txt | 4 +- 7 files changed, 461 insertions(+), 77 deletions(-) delete mode 100644 bibliotecas.list create mode 100644 bibliotecas.py create mode 100644 prompt.py diff --git a/README.md b/README.md index ba4e0f8..588a61c 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ Este projeto é um scraper (raspador de dados) para verificar as bibliotecas Python que tiveram atualização no último mês e listá-las, facilitando a identificação de bibliotecas com lançamentos de versão major. +Este Scraper utiliza IA para que não seja necessário a sua atualização constante + # Como utilizar Para criar um ambiente virtual, execute @@ -19,9 +21,41 @@ Para instalar as dependências do projeto no ambiente virtual, execute pip install -r requirements.txt ``` -Para rodar o scrapper, execute +Para completar a instalação do Crawl4AI +``` +# Install the package +pip install -U crawl4ai + +# Run post-installation setup +crawl4ai-setup + +# Verify your installation +crawl4ai-doctor +``` +Crie uma chave junto a COHERE +- https://docs.cohere.com/ +

+ +Atualize o valor em prompt.py +``` +... +os.environ["COHERE_API_KEY"] = "" +... +``` +Instale o Thinker +```cmd +sudo apt-get install python3-tk +``` + +Para procurar por releases, execute o comando abaixo e aguarde as instruções ``` -python getNews.py +python getNews.py releases ``` +Para criar os resumos, execute +``` +python getNews.py slides +``` + + Para desativar o ambiente virtual, execute `deactivate`. diff --git a/bibliotecas.list b/bibliotecas.list deleted file mode 100644 index f274251..0000000 --- a/bibliotecas.list +++ /dev/null @@ -1,13 +0,0 @@ -pandas -scikit-learn -aiohttp -fastapi -numpy -Django -matplotlib -seaborn -scikit-learn -tensorflow -keras -requests -beautifulsoup4 diff --git a/bibliotecas.py b/bibliotecas.py new file mode 100644 index 0000000..2740641 --- /dev/null +++ b/bibliotecas.py @@ -0,0 +1,142 @@ +bibliotecas = { + "Requests": { + "library_name": "Requests", + "releases_url": "https://pypi.org/project/requests/", + "logo": "https://requests.readthedocs.io/en/latest/_static/requests-sidebar.png", + }, + "Scikit-learn": { + "library_name": "Scikit-learn", + "releases_url": "https://pypi.org/project/scikit-learn/", + "logo": "https://scikit-learn.org/stable/_static/scikit-learn-logo-small.png", + }, + "Numpy": { + "library_name": "Numpy", + "releases_url": "https://pypi.org/project/numpy/", + "logo": "https://numpy.org/devdocs/_static/numpylogo.svg", + }, + "MatPlotLib": { + "library_name": "MatPlotLib", + "releases_url": "https://pypi.org/project/matplotlib/", + "logo": "https://matplotlib.org/stable/_static/logo_light.svg", + }, + "AIOHttp": { + "library_name": "AIOHttp", + "releases_url": "https://pypi.org/project/aiohttp/", + "logo": "https://docs.aiohttp.org/en/stable/_static/aiohttp-plain.svg", + }, + "Pandas": { + "library_name": "Pandas", + "releases_url": "https://pypi.org/project/pandas/", + "logo": "https://pandas.pydata.org/static/img/pandas_mark.svg", + }, + "FastAPI": { + "library_name": "FastAPI", + "releases_url": "https://pypi.org/project/fastapi/", + "logo": "https://fastapi.tiangolo.com/img/icon.png", + }, + "Django": { + "library_name": "Django", + "releases_url": "https://pypi.org/project/Django/", + "logo": "https://static.djangoproject.com/img/logos/django-logo-negative.png", + }, + "Seaborn": { + "library_name": "Seaborn", + "releases_url": "https://pypi.org/project/seaborn/", + "logo": "https://seaborn.pydata.org/_images/logo-wide-lightbg.svg", + }, + "TensorFlow": { + "library_name": "TensorFlow", + "releases_url": "https://pypi.org/project/tensorflow/", + "logo": "https://www.tensorflow.org/images/tf_logo_social.png", + }, + "Keras": { + "library_name": "Keras", + "releases_url": "https://pypi.org/project/keras/", + "logo": "https://keras.io/img/logo.png", + }, + "PyTorch": { + "library_name": "PyTorch", + "releases_url": "https://pypi.org/project/torch/", + "logo": "https://pytorch.org/assets/images/pytorch-logo.png", + }, + "SQLAlchemy": { + "library_name": "SQLAlchemy", + "releases_url": "https://pypi.org/project/SQLAlchemy/", + "logo": "https://www.sqlalchemy.org/img/sqla_logo.png", + }, + "BeaultifulSoup": { + "library_name": "BeaultifulSoup", + "releases_url": "https://pypi.org/project/beautifulsoup4/", + "logo": "https://www.crummy.com/software/BeautifulSoup/10.1.jpg", + }, + "LangChain": { + "library_name": "LangChain", + "releases_url": "https://pypi.org/project/langchain/", + "logo": "https://python.langchain.com/img/brand/wordmark-dark.png", + }, + "CrewAI": { + "library_name": "CrewAI", + "releases_url": "https://pypi.org/project/crewai/", + "logo": "https://cdn.prod.website-files.com/66cf2bfc3ed15b02da0ca770/66d07240057721394308addd_Logo%20(1).svg", + }, + "Flask": { + "library_name": "Flask", + "releases_url": "https://pypi.org/project/Flask/", + "logo": "https://flask.palletsprojects.com/en/stable/_static/flask-vertical.png", + }, + "Pygame": { + "library_name": "Pygame", + "releases_url": "https://pypi.org/project/pygame/", + "logo": "https://www.pygame.org/images/logo_lofi.png", + }, + "Thinker": { + "library_name": "Thinker", + "releases_url": "https://pypi.org/project/thinker/", + "logo": "https://keras.io/img/logo.png", + }, + "Plotly": { + "library_name": "Plotly", + "releases_url": "https://pypi.org/project/plotly/", + "logo": "https://plotly.com/static/img/logos/plotly-logomark.svg", + }, + "MlForecast": { + "library_name": "MlForecast", + "releases_url": "https://pypi.org/project/mlforecast/", + "logo": "https://raw.githubusercontent.com/Nixtla/mlforecast/main/nbs/figs/logo.png", + }, + "GeoPandas": { + "library_name": "GeoPandas", + "releases_url": "https://pypi.org/project/geopandas/", + "logo": "https://geopandas.org/en/stable/_static/geopandas_logo_web.svg", + }, + "AirFlow": { + "library_name": "AirFlow", + "releases_url": "https://pypi.org/project/apache-airflow/", + "logo": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/de/AirflowLogo.png/800px-AirflowLogo.png?20191014185111", + }, + "PySpark": { + "library_name": "PySpark", + "releases_url": "https://pypi.org/project/pyspark/", + "logo": "https://spark.apache.org/docs/latest/api/python/_static/spark-logo-reverse.png", + }, + "Gym": { + "library_name": "Gym", + "releases_url": "https://pypi.org/project/gym/", + "logo": "https://www.gymlibrary.dev/_static/img/gym_logo_black.svg", + }, + "HyperOpt": { + "library_name": "HyperOpt", + "releases_url": "https://pypi.org/project/hyperopt/", + "logo": "https://camo.githubusercontent.com/d9cabe82cdc7bff598f84d61b0a8921cd5c3ceb0716b03399fc31db1a2a23182/68747470733a2f2f692e706f7374696d672e63632f54506d66665772702f68797065726f70742d6e65772e706e67", + }, + "Streamlit": { + "library_name": "Streamlit", + "releases_url": "https://pypi.org/project/streamlit/", + "logo": "https://streamlit.io/images/brand/streamlit-mark-color.png", + }, + "Crawl4ai": { + "library_name": "Crawl4ai", + "releases_url": "https://crawl4ai.com/mkdocs/blog/", + "logo": "https://star-history.com/#unclecode/crawl4ai&Date", + }, +} diff --git a/cacheVariables.py b/cacheVariables.py index 066a145..53bd1e5 100644 --- a/cacheVariables.py +++ b/cacheVariables.py @@ -1 +1 @@ -pynews = {} \ No newline at end of file +pynews = {} diff --git a/getNews.py b/getNews.py index 25effef..d5dac89 100644 --- a/getNews.py +++ b/getNews.py @@ -1,74 +1,158 @@ -from json import dump -from datetime import datetime import asyncio -from bs4 import BeautifulSoup as bs -from babel.dates import format_datetime -import aiohttp +import json +import sys +import time +from datetime import datetime +from json import dump +from tkinter import messagebox + +from crawl4ai import ( + AsyncWebCrawler, + BrowserConfig, + CacheMode, + CrawlerRunConfig, + DefaultMarkdownGenerator, + PruningContentFilter, +) +from bibliotecas import bibliotecas from cacheVariables import pynews +from prompt import Smart +md_generator = DefaultMarkdownGenerator( + content_filter=PruningContentFilter(), + options={ + "ignore_links": True, + "escape_html": False, + "ignore_images": True, + "skip_internal_links": True, + }, +) +config = CrawlerRunConfig( + cache_mode=CacheMode.BYPASS, + markdown_generator=md_generator, +) -async def fetch(session, url): - async with session.get(f"https://pypi.org/project/{url.lower()}/#history") as response: - body = await response.text() - html = bs(body, "html.parser") - title = html.title.decode() - title = title[title.find(">") + 1: title.find(" ")].title() - div_release = html.find( - "div", - {"class": "release release--latest release--current"} +news_json_file = "pynews.json" + + +class PyNews: + def __init__(self): + self.study_case = bibliotecas.copy() + + async def fetch(self, lib, url_name): + browser_config = BrowserConfig( + browser_type="chromium", + headless=True, + text_mode=True, + light_mode=True, + extra_args=["--disable-extensions"], ) - if div_release: - p_date = div_release.find( - "p", - {"class": "release__version-date"} + async with AsyncWebCrawler( + config=browser_config, verbose=True + ) as crawler: + html = await crawler.arun( + url=lib[url_name], + excluded_tags=["form", "header", "footer", "nav"], + exclude_social_media_links=True, + exclude_external_images=True, + config=config, ) + new_html = html.markdown_v2 - p_ver = div_release.find( - "p", - {"class": "release__version"} - ) - else: - return - - if p_date and p_ver: - str_release = p_date.text.strip() - str_version = p_ver.text.strip() - else: - return - - last_release = datetime.strptime(str_release, "%b %d, %Y") - if last_release.month == datetime.now().month: - async with session.get(f"https://pypi.org/project/{url.lower()}/{str_version}") as ver_page: - body = await ver_page.text() - html = bs(body, "html.parser") - project_page = html.find( - "a", {"class": "vertical-tabs__tab vertical-tabs__tab--with-icon vertical-tabs__tab--condensed"})["href"] - pynews[title] = { - "release": format_datetime( - last_release, - format="dd.MMMM.yyyy", - locale="pt_BR" - ).title().replace(".", " de "), - "version": str_version, - "project_page": project_page - } - - -with open("bibliotecas.list") as f: - libs = f.read().split("\n") + response = Smart().answer(url_name, lib, new_html.fit_markdown) + release_date = response.get("release_date") -async def main(): - async with aiohttp.ClientSession() as session: - tasks = [asyncio.ensure_future(fetch(session, url)) for url in libs] + if response.get("release_date"): + release_date = datetime.strptime(release_date, "%Y-%m-%d") + if (datetime.now() - release_date).days <= 30: + response["releases_doc_url"] = ( + "" + ) + response["library_name"] = lib["library_name"] + pynews[lib["library_name"]] = response + elif response.get("news"): + pynews[lib["library_name"]] = response + pynews[lib["library_name"]]["logo"] = bibliotecas[ + lib["library_name"] + ]["logo"] + pynews[lib["library_name"]]["version"] = lib["version"] + pynews[lib["library_name"]]["release_date"] = lib[ + "release_date" + ] + pynews[lib["library_name"]]["releases_doc_url"] = lib[ + "releases_doc_url" + ] + + async def main_loop(self, url_name, list_libs): + tasks = [ + asyncio.ensure_future(self.fetch(self.study_case[lib], url_name)) + for lib in list_libs + ] await asyncio.gather(*tasks) -loop = asyncio.new_event_loop() -asyncio.set_event_loop(loop) -loop.run_until_complete(main()) + async def get(self, url_name): + for i in range(0, len(self.study_case), 10): + batch = { + k: self.study_case[k] + for k in list(self.study_case.keys())[i : i + 10] + } + await self.main_loop(url_name, list(batch.keys())) + time.sleep( + 60 + ) # O Cohere gratuíto permite somente 10 requisições por minuto + + +async def main(): + news = PyNews() + await news.get("releases_url") + with open(news_json_file, "w", encoding="utf-8") as f: + dump(pynews, f) + print( + f"\033[1;37;44m\033[1;30m\n Edite o arquivo {news_json_file} \ +para adicionar as urls contendo o descritivo das novas releases \n" + ) + messagebox.showwarning( + "Aviso", + f"Edite o arquivo {news_json_file} para adicionar as urls contendo \ +o descritivo das novas releases", + icon="warning", + ) + -with open("pynews.json", "w") as f: - dump(pynews, f) +async def slides(): + print("slides") + news = PyNews() + with open(news_json_file, "r", encoding="utf-8") as f: + news.study_case = json.load(f) + await news.get("releases_doc_url") + with open(news_json_file, "w", encoding="utf-8") as f: + dump(pynews, f) + + +if __name__ == "__main__": + if sys.argv[1] == "-h" or sys.argv[1] == "--help": + print("\n \n") + print( + "\033[1;37;42m\033[1;30mCommand: python getNews.py releases - \ +Gera lista das bibliotecas atualizadas nos últimos \ +30 dias\033[0m" + ) + print("\n") + print( + "\033[1;37;44m\033[1;30mCommand: python getNews.py slides - \ +Gerar slides a partir da release_url do arquivo \ +pynews.json\033[0m" + ) + print("\n \n") + sys.exit(0) -print("Pynews executado com sucesso!") + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + if sys.argv[1] == "releases": + loop.run_until_complete(main()) + elif sys.argv[1] == "slides": + loop.run_until_complete(slides()) + loop.close() + sys.exit(0) diff --git a/prompt.py b/prompt.py new file mode 100644 index 0000000..f39a8ba --- /dev/null +++ b/prompt.py @@ -0,0 +1,135 @@ +import json +import os +from enum import Enum +from typing import List, Optional + +import cohere +import pydantic + +os.environ["COHERE_API_KEY"] = "" < YOUR - API - KEY > "" + + +class Tags(str, Enum): + bug_fix = "bug_fix" + updates = "updates" + security_fix = "security_fix" + new_feature = "new_feature" + + def description(self): + descriptions = { + "bug_fix": "Correções de bugs", + "updates": "Atualizações na nova versão", + "security_fix": "Correções de segurança", + "new_feature": "Novas funcionalidades", + } + return descriptions.get(self.value, "Descrição não disponível") + + +class News(pydantic.BaseModel): + description: str = pydantic.Field( + ..., description="The description of the news." + ) + tag: Tags = pydantic.Field(..., description="The tag of the news.") + + +class Resume(pydantic.BaseModel): + """ + Resume class represents a summary of a library with its name, bug fixes, updates, security fixes, and new features. + + """ + + library_name: str = pydantic.Field( + ..., description="The name of the library." + ) + + news: List[News] = pydantic.Field( + ..., description="List of news of the library." + ) + + +class Smart: + def __init__(self): + self.llm = cohere.ClientV2(api_key=os.environ["COHERE_API_KEY"]) + + def answer(self, path, lib, html): + if path == "releases_url": + return self.find_release(lib["library_name"], html) + elif path == "releases_doc_url": + return self.create_resume(lib, html) + + def query(self, prompt): + return self.llm.chat( + model="command-r-plus-08-2024", + messages=[ + { + "role": "user", + "content": prompt, + } + ], + ) + + def find_release(self, lib, html): + prompt = f""" + Com base no seguinte documento Markdown: + + ```markdown + {html} + ``` + + Extraia a versão mais recente e a data de lançamento da biblioteca {lib}. + + Formate a resposta como um objeto JSON com as seguintes chaves: + * `version`: (string) A versão mais recente. Use `null` se não encontrada. + * `release_date`: (string no formato AAAA-MM-DD) A data de lançamento. Use `null` se não encontrada. + Responda somente com o formato JSON. + + Exemplo de formato JSON: + + ```json + {{ + "version": "1.2.3", + "release_date": "2024-10-26" + }} + ``` + + Se a informação não estiver disponível no documento, retorne: + + ```json + {{ + "version": null, + "release_date": null + }} + ``` + """ + return self.reply(prompt) + + def create_resume(self, lib, html): + prompt = f""" + Com base no documento Markdown: + + ```markdown + {html} + ``` + Faça: + - Resumo das novidades da biblioteca {lib["library_name"]} na versão {lib["version"]} separados por categoria de acordo com o schema fornecido. + - Caso alguma novidade for repetida na mesma categoria, liste apenas uma vez. + - Não é necessário preencher todas as categorias, apenas as que possuem informações. + - Se a informação não estiver disponível, retorne None para todos os campos, exceto para `library_name`. + + Schema Pydantic para a resposta: + {Resume.model_json_schema()} + """ + return self.reply(prompt) + + def reply(self, prompt): + + response = self.query(prompt).message.content[0].text + response = response.replace("```json", "") + response = response.replace("```", "") + response = response.replace("\n", "") + + try: + return json.loads(response) + except json.JSONDecodeError: + print("Erro ao decodificar a resposta JSON:", response) + return {} diff --git a/requirements.txt b/requirements.txt index 19fcb7f..9c243c3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,5 @@ beautifulsoup4==4.12.3 babel==2.15.0 -aiohttp==3.9.5 \ No newline at end of file +cohere==5.13.3 +crawl4ai==0.4.247 +tk==0.1.0 \ No newline at end of file From afbe7828efcbed101bb18489639418e881194f19 Mon Sep 17 00:00:00 2001 From: Daniel Soares Martins Date: Fri, 24 Jan 2025 17:27:05 -0300 Subject: [PATCH 2/4] Add Docker support with Dockerfile and docker-compose.yaml; restructure project files and update requirements --- .gitignore | 5 ++ Dockerfile | 22 ++++++++ README.md | 64 +++++++++++----------- app/__init__.py | 0 bibliotecas.py => app/bibliotecas.py | 33 ++++++++++- cacheVariables.py => app/cacheVariables.py | 0 getNews.py => app/getNews.py | 39 ++++++++----- prompt.py => app/prompt.py | 33 +++++++---- docker-compose.yaml | 6 ++ requirements.txt | 4 +- 10 files changed, 148 insertions(+), 58 deletions(-) create mode 100644 Dockerfile create mode 100644 app/__init__.py rename bibliotecas.py => app/bibliotecas.py (73%) rename cacheVariables.py => app/cacheVariables.py (100%) rename getNews.py => app/getNews.py (83%) rename prompt.py => app/prompt.py (77%) create mode 100644 docker-compose.yaml diff --git a/.gitignore b/.gitignore index 5392bd5..80e6ea9 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,11 @@ __pycache__/ *.py[cod] *$py.class +#PynewFiles +pynews* +format.json +export.py + # C extensions *.so diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..a461a6e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,22 @@ +FROM ubuntu:22.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + python3-pip \ + python3.10 \ + tzdata && \ + ln -fs /usr/share/zoneinfo/America/New_York /etc/localtime && \ + dpkg-reconfigure --frontend noninteractive tzdata && \ + apt-get clean && rm -rf /var/lib/apt/lists/* &&\ + mkdir -p app && \ + pip3 install -U crawl4ai && \ + playwright install && \ + playwright install-deps + +WORKDIR /app + +COPY ./app/. . +COPY requirements.txt . + +RUN pip3 install -r requirements.txt \ No newline at end of file diff --git a/README.md b/README.md index 588a61c..ff2d108 100644 --- a/README.md +++ b/README.md @@ -6,33 +6,32 @@ Este Scraper utiliza IA para que não seja necessário a sua atualização const # Como utilizar -Para criar um ambiente virtual, execute -``` -python -m venv .venv -``` +## Instalação do Docker e Docker Compose -Para ativar o ambiente virtual, execute -``` -source .venv/bin/activate -``` +### Ubuntu -Para instalar as dependências do projeto no ambiente virtual, execute -``` -pip install -r requirements.txt -``` +1. **Instale o Docker:** -Para completar a instalação do Crawl4AI -``` -# Install the package -pip install -U crawl4ai + https://www.digitalocean.com/community/tutorials/how-to-install-and-use-docker-on-ubuntu-22-04 -# Run post-installation setup -crawl4ai-setup +2. **Instale o Docker Compose:** + + https://www.digitalocean.com/community/tutorials/how-to-install-and-use-docker-compose-on-ubuntu-22-04 + + +### Windows + +1. **Instale o Docker:** + + https://docs.docker.com/desktop/setup/install/windows-install/ + +2. **Instale o Docker Compose:** + + https://docs.docker.com/compose/install/ + + +### Crie uma chave junto a COHERE -# Verify your installation -crawl4ai-doctor -``` -Crie uma chave junto a COHERE - https://docs.cohere.com/

@@ -42,20 +41,23 @@ Atualize o valor em prompt.py os.environ["COHERE_API_KEY"] = "" ... ``` -Instale o Thinker -```cmd -sudo apt-get install python3-tk + +## Construa o container +```sh +docker compose build ``` -Para procurar por releases, execute o comando abaixo e aguarde as instruções +## Para procurar por releases, execute o comando abaixo e aguarde as instruções ``` -python getNews.py releases +docker compose run --rm --remove-orphans pynews python3 /app/app/getNews.py releases ``` -Para criar os resumos, execute +## Para criar os resumos, execute ``` -python getNews.py slides +docker compose run --rm --remove-orphans pynews python3 /app/app/getNews.py slides ``` +

+

+

- -Para desativar o ambiente virtual, execute `deactivate`. +## Esse é um script que depende da COHERE AI, essa IA assim como todas as outras, ainda não apresentam comportamento estável, portanto esse script deve ser usado sob supervisão diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bibliotecas.py b/app/bibliotecas.py similarity index 73% rename from bibliotecas.py rename to app/bibliotecas.py index 2740641..ded6010 100644 --- a/bibliotecas.py +++ b/app/bibliotecas.py @@ -3,126 +3,151 @@ "library_name": "Requests", "releases_url": "https://pypi.org/project/requests/", "logo": "https://requests.readthedocs.io/en/latest/_static/requests-sidebar.png", + "repository": "https://github.com/psf/requests", }, "Scikit-learn": { "library_name": "Scikit-learn", "releases_url": "https://pypi.org/project/scikit-learn/", "logo": "https://scikit-learn.org/stable/_static/scikit-learn-logo-small.png", + "repository": "https://github.com/scikit-learn/scikit-learn", }, "Numpy": { "library_name": "Numpy", "releases_url": "https://pypi.org/project/numpy/", "logo": "https://numpy.org/devdocs/_static/numpylogo.svg", + "repository": "https://github.com/numpy/numpy", }, "MatPlotLib": { "library_name": "MatPlotLib", "releases_url": "https://pypi.org/project/matplotlib/", "logo": "https://matplotlib.org/stable/_static/logo_light.svg", + "repository": "https://github.com/matplotlib/matplotlib", }, "AIOHttp": { "library_name": "AIOHttp", "releases_url": "https://pypi.org/project/aiohttp/", "logo": "https://docs.aiohttp.org/en/stable/_static/aiohttp-plain.svg", + "repository": "https://github.com/aio-libs/aiohttp", }, "Pandas": { "library_name": "Pandas", "releases_url": "https://pypi.org/project/pandas/", "logo": "https://pandas.pydata.org/static/img/pandas_mark.svg", + "repository": "https://github.com/pandas-dev/pandas", }, "FastAPI": { "library_name": "FastAPI", "releases_url": "https://pypi.org/project/fastapi/", - "logo": "https://fastapi.tiangolo.com/img/icon.png", + "logo": "https://camo.githubusercontent.com/4ebb06d037b495f2c4c67e0ee4599f747e94e6323ece758a7da27fbbcb411250/68747470733a2f2f666173746170692e7469616e676f6c6f2e636f6d2f696d672f6c6f676f2d6d617267696e2f6c6f676f2d7465616c2e706e67", + "repository": "https://github.com/fastapi/fastapi", }, "Django": { "library_name": "Django", "releases_url": "https://pypi.org/project/Django/", "logo": "https://static.djangoproject.com/img/logos/django-logo-negative.png", + "repository": "https://github.com/django/django", }, "Seaborn": { "library_name": "Seaborn", "releases_url": "https://pypi.org/project/seaborn/", "logo": "https://seaborn.pydata.org/_images/logo-wide-lightbg.svg", + "repository": "https://github.com/mwaskom/seaborn", }, "TensorFlow": { "library_name": "TensorFlow", "releases_url": "https://pypi.org/project/tensorflow/", "logo": "https://www.tensorflow.org/images/tf_logo_social.png", + "repository": "https://github.com/tensorflow/tensorflow", }, "Keras": { "library_name": "Keras", "releases_url": "https://pypi.org/project/keras/", "logo": "https://keras.io/img/logo.png", + "repository": "https://github.com/keras-team/keras", }, "PyTorch": { "library_name": "PyTorch", "releases_url": "https://pypi.org/project/torch/", "logo": "https://pytorch.org/assets/images/pytorch-logo.png", + "repository": "https://github.com/pytorch/pytorch", }, "SQLAlchemy": { "library_name": "SQLAlchemy", "releases_url": "https://pypi.org/project/SQLAlchemy/", "logo": "https://www.sqlalchemy.org/img/sqla_logo.png", + "repository": "https://github.com/sqlalchemy/sqlalchemy", }, "BeaultifulSoup": { "library_name": "BeaultifulSoup", "releases_url": "https://pypi.org/project/beautifulsoup4/", "logo": "https://www.crummy.com/software/BeautifulSoup/10.1.jpg", + "repository": None, }, "LangChain": { "library_name": "LangChain", "releases_url": "https://pypi.org/project/langchain/", "logo": "https://python.langchain.com/img/brand/wordmark-dark.png", + "repository": "https://github.com/langchain-ai/langchain", }, "CrewAI": { "library_name": "CrewAI", "releases_url": "https://pypi.org/project/crewai/", "logo": "https://cdn.prod.website-files.com/66cf2bfc3ed15b02da0ca770/66d07240057721394308addd_Logo%20(1).svg", + "repository": "https://github.com/crewAIInc/crewAI", }, "Flask": { "library_name": "Flask", "releases_url": "https://pypi.org/project/Flask/", "logo": "https://flask.palletsprojects.com/en/stable/_static/flask-vertical.png", + "repository": "https://github.com/pallets/flask", }, "Pygame": { "library_name": "Pygame", "releases_url": "https://pypi.org/project/pygame/", "logo": "https://www.pygame.org/images/logo_lofi.png", + "repository": "https://github.com/pygame/pygame", }, "Thinker": { "library_name": "Thinker", "releases_url": "https://pypi.org/project/thinker/", "logo": "https://keras.io/img/logo.png", + "repository": "https://github.com/mehmetkose/thinker", }, "Plotly": { "library_name": "Plotly", "releases_url": "https://pypi.org/project/plotly/", "logo": "https://plotly.com/static/img/logos/plotly-logomark.svg", + "repository": "https://github.com/plotly/plotly.py", }, "MlForecast": { "library_name": "MlForecast", "releases_url": "https://pypi.org/project/mlforecast/", "logo": "https://raw.githubusercontent.com/Nixtla/mlforecast/main/nbs/figs/logo.png", + "repository": "https://github.com/Nixtla/mlforecast", }, "GeoPandas": { "library_name": "GeoPandas", "releases_url": "https://pypi.org/project/geopandas/", "logo": "https://geopandas.org/en/stable/_static/geopandas_logo_web.svg", + "repository": "https://github.com/geopandas/geopandas", }, "AirFlow": { "library_name": "AirFlow", "releases_url": "https://pypi.org/project/apache-airflow/", "logo": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/de/AirflowLogo.png/800px-AirflowLogo.png?20191014185111", + "repository": "https://github.com/apache/airflow", }, "PySpark": { "library_name": "PySpark", "releases_url": "https://pypi.org/project/pyspark/", "logo": "https://spark.apache.org/docs/latest/api/python/_static/spark-logo-reverse.png", + "repository": "https://github.com/apache/spark/tree/master/python", }, "Gym": { "library_name": "Gym", "releases_url": "https://pypi.org/project/gym/", "logo": "https://www.gymlibrary.dev/_static/img/gym_logo_black.svg", + "repository": "https://github.com/Farama-Foundation/Gymnasium", }, "HyperOpt": { "library_name": "HyperOpt", @@ -139,4 +164,10 @@ "releases_url": "https://crawl4ai.com/mkdocs/blog/", "logo": "https://star-history.com/#unclecode/crawl4ai&Date", }, + "ScanAPI": { + "library_name": "ScanAPI", + "releases_url": "https://pypi.org/project/scanapi/", + "logo": "https://avatars.githubusercontent.com/u/59395469?s=200&v=4", + "repository": "https://github.com/scanapi/scanapi", + }, } diff --git a/cacheVariables.py b/app/cacheVariables.py similarity index 100% rename from cacheVariables.py rename to app/cacheVariables.py diff --git a/getNews.py b/app/getNews.py similarity index 83% rename from getNews.py rename to app/getNews.py index d5dac89..638741b 100644 --- a/getNews.py +++ b/app/getNews.py @@ -4,8 +4,9 @@ import time from datetime import datetime from json import dump -from tkinter import messagebox +from bibliotecas import bibliotecas +from cacheVariables import pynews from crawl4ai import ( AsyncWebCrawler, BrowserConfig, @@ -14,9 +15,6 @@ DefaultMarkdownGenerator, PruningContentFilter, ) - -from bibliotecas import bibliotecas -from cacheVariables import pynews from prompt import Smart md_generator = DefaultMarkdownGenerator( @@ -60,7 +58,16 @@ async def fetch(self, lib, url_name): ) new_html = html.markdown_v2 - response = Smart().answer(url_name, lib, new_html.fit_markdown) + response = {} + try: + response = Smart().answer(url_name, lib, new_html.fit_markdown) + except Exception: + print("$$$ COHERE AI Time Out $$$") + print("Biblioteca não processada :", lib["library_name"]) + print( + "Tente rodar o script novamente somente com a lib : ", + lib["library_name"], + ) release_date = response.get("release_date") @@ -113,12 +120,6 @@ async def main(): f"\033[1;37;44m\033[1;30m\n Edite o arquivo {news_json_file} \ para adicionar as urls contendo o descritivo das novas releases \n" ) - messagebox.showwarning( - "Aviso", - f"Edite o arquivo {news_json_file} para adicionar as urls contendo \ -o descritivo das novas releases", - icon="warning", - ) async def slides(): @@ -127,8 +128,20 @@ async def slides(): with open(news_json_file, "r", encoding="utf-8") as f: news.study_case = json.load(f) await news.get("releases_doc_url") - with open(news_json_file, "w", encoding="utf-8") as f: - dump(pynews, f) + slides = {} + try: + with open("pynews_slides.json", "r", encoding="utf-8") as f: + slides = json.load(f) + except Exception: + pass + for item in pynews: + slides[item] = pynews[item] + with open("pynews_slides.json", "w", encoding="utf-8") as f: + dump(slides, f) + print( + "\033[1;37;44m\033[1;30m\n Abra o arquivo pynews_slides.json para \ +ter acesso ao conteúdo produzido.\n" + ) if __name__ == "__main__": diff --git a/prompt.py b/app/prompt.py similarity index 77% rename from prompt.py rename to app/prompt.py index f39a8ba..b24ad2f 100644 --- a/prompt.py +++ b/app/prompt.py @@ -6,7 +6,7 @@ import cohere import pydantic -os.environ["COHERE_API_KEY"] = "" < YOUR - API - KEY > "" +os.environ["COHERE_API_KEY"] = "" class Tags(str, Enum): @@ -116,20 +116,33 @@ def create_resume(self, lib, html): - Não é necessário preencher todas as categorias, apenas as que possuem informações. - Se a informação não estiver disponível, retorne None para todos os campos, exceto para `library_name`. - Schema Pydantic para a resposta: - {Resume.model_json_schema()} + Não seja criativo, responda somente com que for encontrado no documento. + Não seja criativo na formatação, responda exatamente como o schema fornecido. + Verifique se sua resposta está no formato JSON para que seja possível usar o comando json.loads do python, Caso não esteja, formate-a corretamente. + + response_format: + {json.dumps(Resume.model_json_schema())} """ return self.reply(prompt) def reply(self, prompt): - response = self.query(prompt).message.content[0].text + response = self.query(prompt) + response = response.message.content[0].text response = response.replace("```json", "") response = response.replace("```", "") response = response.replace("\n", "") - - try: - return json.loads(response) - except json.JSONDecodeError: - print("Erro ao decodificar a resposta JSON:", response) - return {} + _response = {} + one_time = True + while len(response) > 10: + try: + _response = json.loads(response) + break + except json.JSONDecodeError: + if one_time: + print("### COHERE retornou JSON em formato errado") + print(response) + print("### Tentando formatar ###") + one_time = False + response = response[:-1] + return _response diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..d896d5a --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,6 @@ +services: + pynews: + build: . + volumes: + - .:/app + working_dir: /app \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 9c243c3..ac290d5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,3 @@ -beautifulsoup4==4.12.3 babel==2.15.0 cohere==5.13.3 -crawl4ai==0.4.247 -tk==0.1.0 \ No newline at end of file +crawl4ai==0.4.247 \ No newline at end of file From 1bab29321f1c7478c46c8735ab3f073974bc6ee9 Mon Sep 17 00:00:00 2001 From: Daniel Soares Martins Date: Wed, 29 Jan 2025 17:30:54 -0300 Subject: [PATCH 3/4] Update integration with Google Gemini API; replace Cohere references, add jeannie.py for content generation, and modify README accordingly --- .gitignore | 1 + README.md | 8 ++++---- app/bibliotecas.py | 2 +- app/getNews.py | 14 ++++---------- app/jeannie.py | 14 ++++++++++++++ app/prompt.py | 22 ++++++---------------- requirements.txt | 4 ++-- 7 files changed, 32 insertions(+), 33 deletions(-) create mode 100644 app/jeannie.py diff --git a/.gitignore b/.gitignore index 80e6ea9..ccd1eb0 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ __pycache__/ pynews* format.json export.py +worker.py # C extensions *.so diff --git a/README.md b/README.md index ff2d108..ee55fd7 100644 --- a/README.md +++ b/README.md @@ -30,15 +30,15 @@ Este Scraper utiliza IA para que não seja necessário a sua atualização const https://docs.docker.com/compose/install/ -### Crie uma chave junto a COHERE +### Crie uma chave GEMINI API junto a Google -- https://docs.cohere.com/ +- https://aistudio.google.com/

-Atualize o valor em prompt.py +Atualize o valor em jeannie.py ``` ... -os.environ["COHERE_API_KEY"] = "" +GOOGLE_API_KEY = "" ... ``` diff --git a/app/bibliotecas.py b/app/bibliotecas.py index ded6010..6e8e476 100644 --- a/app/bibliotecas.py +++ b/app/bibliotecas.py @@ -44,7 +44,7 @@ "Django": { "library_name": "Django", "releases_url": "https://pypi.org/project/Django/", - "logo": "https://static.djangoproject.com/img/logos/django-logo-negative.png", + "logo": "https://www.djangoproject.com/m/img/logos/django-logo-positive.png", "repository": "https://github.com/django/django", }, "Seaborn": { diff --git a/app/getNews.py b/app/getNews.py index 638741b..8577ce4 100644 --- a/app/getNews.py +++ b/app/getNews.py @@ -1,7 +1,6 @@ import asyncio import json import sys -import time from datetime import datetime from json import dump @@ -81,6 +80,9 @@ async def fetch(self, lib, url_name): pynews[lib["library_name"]] = response elif response.get("news"): pynews[lib["library_name"]] = response + pynews[lib["library_name"]]["library_name"] = lib[ + "library_name" + ] pynews[lib["library_name"]]["logo"] = bibliotecas[ lib["library_name"] ]["logo"] @@ -100,15 +102,7 @@ async def main_loop(self, url_name, list_libs): await asyncio.gather(*tasks) async def get(self, url_name): - for i in range(0, len(self.study_case), 10): - batch = { - k: self.study_case[k] - for k in list(self.study_case.keys())[i : i + 10] - } - await self.main_loop(url_name, list(batch.keys())) - time.sleep( - 60 - ) # O Cohere gratuíto permite somente 10 requisições por minuto + await self.main_loop(url_name, list(self.study_case.keys())) async def main(): diff --git a/app/jeannie.py b/app/jeannie.py new file mode 100644 index 0000000..87850d6 --- /dev/null +++ b/app/jeannie.py @@ -0,0 +1,14 @@ +import google.generativeai as genai + +GOOGLE_API_KEY = "" + +genai.configure(api_key=GOOGLE_API_KEY) + +model = genai.GenerativeModel("gemini-1.5-flash") + + +def generate_content(prompt): + + response = model.generate_content(prompt) + + return response.text diff --git a/app/prompt.py b/app/prompt.py index b24ad2f..8adde17 100644 --- a/app/prompt.py +++ b/app/prompt.py @@ -1,12 +1,12 @@ import json import os from enum import Enum -from typing import List, Optional +from typing import List -import cohere import pydantic +from jeannie import generate_content -os.environ["COHERE_API_KEY"] = "" +os.environ["COHERE_API_KEY"] = "niT0L5XZyimfgvLYhzviA3Xs11cHSNMjkwzzQ9OB" class Tags(str, Enum): @@ -48,9 +48,6 @@ class Resume(pydantic.BaseModel): class Smart: - def __init__(self): - self.llm = cohere.ClientV2(api_key=os.environ["COHERE_API_KEY"]) - def answer(self, path, lib, html): if path == "releases_url": return self.find_release(lib["library_name"], html) @@ -58,15 +55,7 @@ def answer(self, path, lib, html): return self.create_resume(lib, html) def query(self, prompt): - return self.llm.chat( - model="command-r-plus-08-2024", - messages=[ - { - "role": "user", - "content": prompt, - } - ], - ) + return generate_content(prompt) def find_release(self, lib, html): prompt = f""" @@ -120,6 +109,8 @@ def create_resume(self, lib, html): Não seja criativo na formatação, responda exatamente como o schema fornecido. Verifique se sua resposta está no formato JSON para que seja possível usar o comando json.loads do python, Caso não esteja, formate-a corretamente. + Sempre responda no idioma Português do Brasil + response_format: {json.dumps(Resume.model_json_schema())} """ @@ -128,7 +119,6 @@ def create_resume(self, lib, html): def reply(self, prompt): response = self.query(prompt) - response = response.message.content[0].text response = response.replace("```json", "") response = response.replace("```", "") response = response.replace("\n", "") diff --git a/requirements.txt b/requirements.txt index ac290d5..e8897ea 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ babel==2.15.0 -cohere==5.13.3 -crawl4ai==0.4.247 \ No newline at end of file +crawl4ai==0.4.247 +google-generativeai==0.8.4 \ No newline at end of file From 2fce624ae5d91b4a1690681839f55d722f4c15d1 Mon Sep 17 00:00:00 2001 From: Daniel Soares Martins Date: Tue, 6 May 2025 15:20:08 -0300 Subject: [PATCH 4/4] Add Docker support with .dockerignore and Dockerfile; restructure project for multi-stage builds and add Poetry for dependency management --- .dockerignore | 9 + Dockerfile | 28 ++- README.md | 2 + app/bibliotecas.py | 84 ++++++++- app/getNews.py | 32 +++- app/jeannie.py | 19 +- app/prompt.py | 15 +- poetry.lock | 455 +++++++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 16 ++ requirements.txt | 3 +- 10 files changed, 634 insertions(+), 29 deletions(-) create mode 100644 .dockerignore create mode 100644 poetry.lock create mode 100644 pyproject.toml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..56d8d5f --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +.git +__pycache__ +*.pyc +tests/ +docs/ +.venv/ +node_modules/ +.env/ +# ... other files/directories to exclude \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index a461a6e..f3985b1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,12 @@ -FROM ubuntu:22.04 +FROM python:3.12.2-slim as builder -ENV DEBIAN_FRONTEND=noninteractive +WORKDIR /app + +ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONUNBUFFERED 1 -RUN apt-get update && apt-get install -y \ - python3-pip \ - python3.10 \ +RUN apt-get update && \ + apt-get install -y --no-install-recommends gcc \ tzdata && \ ln -fs /usr/share/zoneinfo/America/New_York /etc/localtime && \ dpkg-reconfigure --frontend noninteractive tzdata && \ @@ -14,9 +16,17 @@ RUN apt-get update && apt-get install -y \ playwright install && \ playwright install-deps -WORKDIR /app - -COPY ./app/. . COPY requirements.txt . +RUN pip wheel --no-cache-dir --no-deps --wheel-dir /app/wheels -r requirements.txt + + +# final stage +FROM python:3.12.2-slim + +WORKDIR /app -RUN pip3 install -r requirements.txt \ No newline at end of file +COPY --from=builder /app/wheels /wheels +COPY --from=builder /app/requirements.txt . +RUN pip install --no-cache /wheels/* +RUN playwright install +RUN playwright install-deps \ No newline at end of file diff --git a/README.md b/README.md index ee55fd7..9c47971 100644 --- a/README.md +++ b/README.md @@ -61,3 +61,5 @@ docker compose run --rm --remove-orphans pynews python3 /app/app/getNews.py slid

## Esse é um script que depende da COHERE AI, essa IA assim como todas as outras, ainda não apresentam comportamento estável, portanto esse script deve ser usado sob supervisão + +## DeepSeek "This model's maximum context length is 65536 tokens" diff --git a/app/bibliotecas.py b/app/bibliotecas.py index 6e8e476..48b4b57 100644 --- a/app/bibliotecas.py +++ b/app/bibliotecas.py @@ -4,30 +4,52 @@ "releases_url": "https://pypi.org/project/requests/", "logo": "https://requests.readthedocs.io/en/latest/_static/requests-sidebar.png", "repository": "https://github.com/psf/requests", + "fixed_release_url": "https://requests.readthedocs.io/en/latest/community/updates/#release-history", }, "Scikit-learn": { "library_name": "Scikit-learn", "releases_url": "https://pypi.org/project/scikit-learn/", "logo": "https://scikit-learn.org/stable/_static/scikit-learn-logo-small.png", "repository": "https://github.com/scikit-learn/scikit-learn", + "releases_urls_list": [ + "https://scikit-learn.org/stable/whats_new/v1.5.html#version-1-5-0", + "https://scikit-learn.org/stable/whats_new/v1.5.html#version-1-5-1", + "https://scikit-learn.org/stable/whats_new/v1.5.html#version-1-5-2", + "https://scikit-learn.org/stable/whats_new/v1.6.html#version-1-6-0", + "https://scikit-learn.org/stable/whats_new/v1.6.html#version-1-6-1", + ], }, "Numpy": { "library_name": "Numpy", "releases_url": "https://pypi.org/project/numpy/", "logo": "https://numpy.org/devdocs/_static/numpylogo.svg", "repository": "https://github.com/numpy/numpy", + "releases_urls_list": [ + "https://github.com/numpy/numpy/releases/tag/v2.2.1", + "https://github.com/numpy/numpy/releases/tag/v2.2.2", + "https://github.com/numpy/numpy/releases/tag/v2.2.3", + "https://github.com/numpy/numpy/releases/tag/v2.2.4", + ], }, "MatPlotLib": { "library_name": "MatPlotLib", "releases_url": "https://pypi.org/project/matplotlib/", "logo": "https://matplotlib.org/stable/_static/logo_light.svg", "repository": "https://github.com/matplotlib/matplotlib", + "releases_urls_list": [ + "https://matplotlib.org/stable/api/prev_api_changes/api_changes_3.9.0.html", + "https://matplotlib.org/stable/api/prev_api_changes/api_changes_3.9.1.html", + "https://matplotlib.org/stable/api/prev_api_changes/api_changes_3.9.2.html", + "https://matplotlib.org/stable/api/prev_api_changes/api_changes_3.10.0.html", + "https://matplotlib.org/stable/api/prev_api_changes/api_changes_3.10.1.html", + ], }, "AIOHttp": { "library_name": "AIOHttp", "releases_url": "https://pypi.org/project/aiohttp/", "logo": "https://docs.aiohttp.org/en/stable/_static/aiohttp-plain.svg", "repository": "https://github.com/aio-libs/aiohttp", + "fixed_release_url": "https://docs.aiohttp.org/en/stable/changes.html", }, "Pandas": { "library_name": "Pandas", @@ -40,134 +62,194 @@ "releases_url": "https://pypi.org/project/fastapi/", "logo": "https://camo.githubusercontent.com/4ebb06d037b495f2c4c67e0ee4599f747e94e6323ece758a7da27fbbcb411250/68747470733a2f2f666173746170692e7469616e676f6c6f2e636f6d2f696d672f6c6f676f2d6d617267696e2f6c6f676f2d7465616c2e706e67", "repository": "https://github.com/fastapi/fastapi", + "fixed_release_url": "https://github.com/fastapi/fastapi/releases", }, "Django": { "library_name": "Django", "releases_url": "https://pypi.org/project/Django/", "logo": "https://www.djangoproject.com/m/img/logos/django-logo-positive.png", "repository": "https://github.com/django/django", + "releases_urls_list": [ + "https://docs.djangoproject.com/en/4.2/releases/4.2/", + "https://docs.djangoproject.com/en/4.2/releases/4.2.1/", + "https://docs.djangoproject.com/en/4.2/releases/4.2.2/", + "https://docs.djangoproject.com/en/5.0/releases/5.0/", + "https://docs.djangoproject.com/en/5.0/releases/5.0.1/", + "https://docs.djangoproject.com/en/5.0/releases/5.0.2/", + "https://docs.djangoproject.com/en/5.1/releases/5.1/", + "https://docs.djangoproject.com/en/5.1/releases/5.1.1/", + "https://docs.djangoproject.com/en/5.1/releases/5.1.2/", + ], }, "Seaborn": { "library_name": "Seaborn", "releases_url": "https://pypi.org/project/seaborn/", "logo": "https://seaborn.pydata.org/_images/logo-wide-lightbg.svg", "repository": "https://github.com/mwaskom/seaborn", + "releases_urls_list": [ + "https://seaborn.pydata.org/whatsnew/v0.12.0.html", + "https://seaborn.pydata.org/whatsnew/v0.12.1.html", + "https://seaborn.pydata.org/whatsnew/v0.12.2.html", + "https://seaborn.pydata.org/whatsnew/v0.13.0.html", + "https://seaborn.pydata.org/whatsnew/v0.13.1.html", + "https://seaborn.pydata.org/whatsnew/v0.13.2.html", + ], }, "TensorFlow": { "library_name": "TensorFlow", "releases_url": "https://pypi.org/project/tensorflow/", "logo": "https://www.tensorflow.org/images/tf_logo_social.png", "repository": "https://github.com/tensorflow/tensorflow", + "fixed_release_url": "https://github.com/tensorflow/tensorflow/releases", }, "Keras": { "library_name": "Keras", "releases_url": "https://pypi.org/project/keras/", "logo": "https://keras.io/img/logo.png", "repository": "https://github.com/keras-team/keras", + "fixed_release_url": "https://github.com/keras-team/keras/releases", }, "PyTorch": { "library_name": "PyTorch", "releases_url": "https://pypi.org/project/torch/", "logo": "https://pytorch.org/assets/images/pytorch-logo.png", "repository": "https://github.com/pytorch/pytorch", + "fixed_release_url": "https://github.com/pytorch/pytorch/releases", }, "SQLAlchemy": { "library_name": "SQLAlchemy", "releases_url": "https://pypi.org/project/SQLAlchemy/", "logo": "https://www.sqlalchemy.org/img/sqla_logo.png", - "repository": "https://github.com/sqlalchemy/sqlalchemy", + "repository": "https://github.com/sqlalchemy/sqlalchemy/releases", + "releases_urls_list": [ + "https://docs.sqlalchemy.org/en/20/changelog/changelog_10.html#change-1.0.0", + "https://docs.sqlalchemy.org/en/20/changelog/changelog_10.html#change-1.0.1", + "https://docs.sqlalchemy.org/en/20/changelog/changelog_10.html#change-1.0.2", + "https://docs.sqlalchemy.org/en/20/changelog/changelog_10.html#change-1.0.3", + "https://docs.sqlalchemy.org/en/20/changelog/changelog_20.html#change-2.0.0", + "https://docs.sqlalchemy.org/en/20/changelog/changelog_20.html#change-2.0.1", + "https://docs.sqlalchemy.org/en/20/changelog/changelog_20.html#change-2.0.2", + ], }, "BeaultifulSoup": { "library_name": "BeaultifulSoup", "releases_url": "https://pypi.org/project/beautifulsoup4/", "logo": "https://www.crummy.com/software/BeautifulSoup/10.1.jpg", "repository": None, + "fixed_release_url": "https://git.launchpad.net/beautifulsoup/tree/CHANGELOG", }, "LangChain": { "library_name": "LangChain", "releases_url": "https://pypi.org/project/langchain/", "logo": "https://python.langchain.com/img/brand/wordmark-dark.png", "repository": "https://github.com/langchain-ai/langchain", + "fixed_release_url": "https://github.com/langchain-ai/langchain/releases", }, "CrewAI": { "library_name": "CrewAI", "releases_url": "https://pypi.org/project/crewai/", "logo": "https://cdn.prod.website-files.com/66cf2bfc3ed15b02da0ca770/66d07240057721394308addd_Logo%20(1).svg", "repository": "https://github.com/crewAIInc/crewAI", + "fixed_release_url": "https://github.com/crewAIInc/crewAI/releases", }, "Flask": { "library_name": "Flask", "releases_url": "https://pypi.org/project/Flask/", "logo": "https://flask.palletsprojects.com/en/stable/_static/flask-vertical.png", "repository": "https://github.com/pallets/flask", + "fixed_release_url": "https://github.com/pallets/flask/releases", }, "Pygame": { "library_name": "Pygame", "releases_url": "https://pypi.org/project/pygame/", "logo": "https://www.pygame.org/images/logo_lofi.png", "repository": "https://github.com/pygame/pygame", + "fixed_release_url": "https://github.com/pygame/pygame/releases", }, "Thinker": { "library_name": "Thinker", "releases_url": "https://pypi.org/project/thinker/", "logo": "https://keras.io/img/logo.png", "repository": "https://github.com/mehmetkose/thinker", + "fixed_release_url": "https://github.com/mehmetkose/thinker/releases", }, "Plotly": { "library_name": "Plotly", "releases_url": "https://pypi.org/project/plotly/", "logo": "https://plotly.com/static/img/logos/plotly-logomark.svg", "repository": "https://github.com/plotly/plotly.py", + "fixed_release_url": "https://github.com/plotly/plotly.py/releases", }, "MlForecast": { "library_name": "MlForecast", "releases_url": "https://pypi.org/project/mlforecast/", "logo": "https://raw.githubusercontent.com/Nixtla/mlforecast/main/nbs/figs/logo.png", "repository": "https://github.com/Nixtla/mlforecast", + "fixed_release_url": "https://github.com/Nixtla/mlforecast/releases", }, "GeoPandas": { "library_name": "GeoPandas", "releases_url": "https://pypi.org/project/geopandas/", "logo": "https://geopandas.org/en/stable/_static/geopandas_logo_web.svg", "repository": "https://github.com/geopandas/geopandas", + "fixed_release_url": "https://github.com/geopandas/geopandas/releases", }, "AirFlow": { "library_name": "AirFlow", "releases_url": "https://pypi.org/project/apache-airflow/", "logo": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/de/AirflowLogo.png/800px-AirflowLogo.png?20191014185111", "repository": "https://github.com/apache/airflow", + "fixed_release_url": "https://github.com/apache/airflow/releases", }, "PySpark": { "library_name": "PySpark", "releases_url": "https://pypi.org/project/pyspark/", "logo": "https://spark.apache.org/docs/latest/api/python/_static/spark-logo-reverse.png", "repository": "https://github.com/apache/spark/tree/master/python", + "releases_urls_list": [ + "https://spark.apache.org/releases/spark-release-3-4-4.html", + "https://spark.apache.org/releases/spark-release-3-5-4.html", + "https://spark.apache.org/releases/spark-release-3-5-5.html", + ], }, "Gym": { "library_name": "Gym", "releases_url": "https://pypi.org/project/gym/", "logo": "https://www.gymlibrary.dev/_static/img/gym_logo_black.svg", "repository": "https://github.com/Farama-Foundation/Gymnasium", + "fixed_release_url": "https://github.com/Farama-Foundation/Gymnasium/releases", }, "HyperOpt": { "library_name": "HyperOpt", "releases_url": "https://pypi.org/project/hyperopt/", "logo": "https://camo.githubusercontent.com/d9cabe82cdc7bff598f84d61b0a8921cd5c3ceb0716b03399fc31db1a2a23182/68747470733a2f2f692e706f7374696d672e63632f54506d66665772702f68797065726f70742d6e65772e706e67", + "fixed_release_url": "https://github.com/hyperopt/hyperopt/releases", }, "Streamlit": { "library_name": "Streamlit", "releases_url": "https://pypi.org/project/streamlit/", "logo": "https://streamlit.io/images/brand/streamlit-mark-color.png", + "repository": "https://github.com/streamlit/streamlit", + "fixed_release_url": "https://docs.streamlit.io/develop/quick-reference/release-notes", }, "Crawl4ai": { "library_name": "Crawl4ai", "releases_url": "https://crawl4ai.com/mkdocs/blog/", "logo": "https://star-history.com/#unclecode/crawl4ai&Date", + "fixed_release_url": "https://github.com/unclecode/crawl4ai/releases", }, "ScanAPI": { "library_name": "ScanAPI", "releases_url": "https://pypi.org/project/scanapi/", "logo": "https://avatars.githubusercontent.com/u/59395469?s=200&v=4", "repository": "https://github.com/scanapi/scanapi", + "fixed_release_url": "https://github.com/scanapi/scanapi/releases", + }, + "Polars": { + "library_name": "Polars", + "releases_url": "https://pypi.org/project/polars/", + "logo": "https://raw.githubusercontent.com/pola-rs/polars-static/master/banner/polars_github_banner.svg", + "repository": "https://github.com/pola-rs/polars?tab=readme-ov-file", + "fixed_release_url": "https://github.com/pola-rs/polars/releases", }, } diff --git a/app/getNews.py b/app/getNews.py index 8577ce4..e1b4590 100644 --- a/app/getNews.py +++ b/app/getNews.py @@ -1,6 +1,7 @@ import asyncio import json import sys +import time from datetime import datetime from json import dump @@ -37,6 +38,14 @@ class PyNews: def __init__(self): self.study_case = bibliotecas.copy() + def check_for_release_url(self, lib): + if lib.get("fixed_release_url", None): + return lib["fixed_release_url"] + else: + return Smart().think_about_release_url( + lib["releases_urls_list"], lib["version"] + ) + async def fetch(self, lib, url_name): browser_config = BrowserConfig( browser_type="chromium", @@ -58,23 +67,20 @@ async def fetch(self, lib, url_name): new_html = html.markdown_v2 response = {} + count = 3 try: response = Smart().answer(url_name, lib, new_html.fit_markdown) - except Exception: - print("$$$ COHERE AI Time Out $$$") + except Exception as e: print("Biblioteca não processada :", lib["library_name"]) - print( - "Tente rodar o script novamente somente com a lib : ", - lib["library_name"], - ) - + print(e) release_date = response.get("release_date") if response.get("release_date"): release_date = datetime.strptime(release_date, "%Y-%m-%d") if (datetime.now() - release_date).days <= 30: - response["releases_doc_url"] = ( - "" + lib["version"] = response["version"] + response["releases_doc_url"] = self.check_for_release_url( + lib ) response["library_name"] = lib["library_name"] pynews[lib["library_name"]] = response @@ -93,6 +99,14 @@ async def fetch(self, lib, url_name): pynews[lib["library_name"]]["releases_doc_url"] = lib[ "releases_doc_url" ] + if bibliotecas[lib["library_name"]].get("fixed_release_url"): + pynews[lib["library_name"]]["fixed_release_url"] = ( + bibliotecas[lib["library_name"]]["fixed_release_url"] + ) + else: + pynews[lib["library_name"]]["releases_urls_list"] = ( + bibliotecas[lib["library_name"]]["releases_urls_list"] + ) async def main_loop(self, url_name, list_libs): tasks = [ diff --git a/app/jeannie.py b/app/jeannie.py index 87850d6..1915f33 100644 --- a/app/jeannie.py +++ b/app/jeannie.py @@ -1,14 +1,17 @@ -import google.generativeai as genai +from openai import OpenAI -GOOGLE_API_KEY = "" - -genai.configure(api_key=GOOGLE_API_KEY) - -model = genai.GenerativeModel("gemini-1.5-flash") +client = OpenAI( + api_key="", + base_url="https://api.deepseek.com", +) def generate_content(prompt): - response = model.generate_content(prompt) + response = client.chat.completions.create( + model="deepseek-chat", + messages=[{"role": "user", "content": prompt}], + stream=False, + ) - return response.text + return response.choices[0].message.content diff --git a/app/prompt.py b/app/prompt.py index 8adde17..3ba0a58 100644 --- a/app/prompt.py +++ b/app/prompt.py @@ -109,13 +109,26 @@ def create_resume(self, lib, html): Não seja criativo na formatação, responda exatamente como o schema fornecido. Verifique se sua resposta está no formato JSON para que seja possível usar o comando json.loads do python, Caso não esteja, formate-a corretamente. - Sempre responda no idioma Português do Brasil + Sempre traduza para o idioma Português do Brasil response_format: {json.dumps(Resume.model_json_schema())} """ return self.reply(prompt) + def think_about_release_url(self, releases_urls_list, new_release_version): + prompt = f""" + Faça: + - Analise a lista de urls fornecidas: + {releases_urls_list} + - Após a análise da lista de urls, de acordo com o padrão das urls, determine qual será a url referente a versão {new_release_version} + - Responda somente com a nova url prevista, não retorne nenhum outro texto. + """ + response = self.query(prompt) + response = response.replace("`", "") + response = response.replace("\n", "") + return response + def reply(self, prompt): response = self.query(prompt) diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 0000000..e6ab518 --- /dev/null +++ b/poetry.lock @@ -0,0 +1,455 @@ +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. + +[[package]] +name = "annotated-types" +version = "0.7.0" +description = "Reusable constraint types to use with typing.Annotated" +optional = false +python-versions = ">=3.8" +files = [ + {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"}, + {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, +] + +[[package]] +name = "anyio" +version = "4.9.0" +description = "High level compatibility layer for multiple asynchronous event loop implementations" +optional = false +python-versions = ">=3.9" +files = [ + {file = "anyio-4.9.0-py3-none-any.whl", hash = "sha256:9f76d541cad6e36af7beb62e978876f3b41e3e04f2c1fbf0884604c0a9c4d93c"}, + {file = "anyio-4.9.0.tar.gz", hash = "sha256:673c0c244e15788651a4ff38710fea9675823028a6f08a5eda409e0c9840a028"}, +] + +[package.dependencies] +idna = ">=2.8" +sniffio = ">=1.1" +typing_extensions = {version = ">=4.5", markers = "python_version < \"3.13\""} + +[package.extras] +doc = ["Sphinx (>=8.2,<9.0)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx_rtd_theme"] +test = ["anyio[trio]", "blockbuster (>=1.5.23)", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "trustme", "truststore (>=0.9.1)", "uvloop (>=0.21)"] +trio = ["trio (>=0.26.1)"] + +[[package]] +name = "babel" +version = "2.17.0" +description = "Internationalization utilities" +optional = false +python-versions = ">=3.8" +files = [ + {file = "babel-2.17.0-py3-none-any.whl", hash = "sha256:4d0b53093fdfb4b21c92b5213dba5a1b23885afa8383709427046b21c366e5f2"}, + {file = "babel-2.17.0.tar.gz", hash = "sha256:0c54cffb19f690cdcc52a3b50bcbf71e07a808d1c80d549f2459b9d2cf0afb9d"}, +] + +[package.extras] +dev = ["backports.zoneinfo", "freezegun (>=1.0,<2.0)", "jinja2 (>=3.0)", "pytest (>=6.0)", "pytest-cov", "pytz", "setuptools", "tzdata"] + +[[package]] +name = "certifi" +version = "2025.4.26" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2025.4.26-py3-none-any.whl", hash = "sha256:30350364dfe371162649852c63336a15c70c6510c2ad5015b21c2345311805f3"}, + {file = "certifi-2025.4.26.tar.gz", hash = "sha256:0a816057ea3cdefcef70270d2c515e4506bbc954f417fa5ade2021213bb8f0c6"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "distro" +version = "1.9.0" +description = "Distro - an OS platform information API" +optional = false +python-versions = ">=3.6" +files = [ + {file = "distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2"}, + {file = "distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed"}, +] + +[[package]] +name = "h11" +version = "0.16.0" +description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" +optional = false +python-versions = ">=3.8" +files = [ + {file = "h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86"}, + {file = "h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1"}, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +description = "A minimal low-level HTTP client." +optional = false +python-versions = ">=3.8" +files = [ + {file = "httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55"}, + {file = "httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8"}, +] + +[package.dependencies] +certifi = "*" +h11 = ">=0.16" + +[package.extras] +asyncio = ["anyio (>=4.0,<5.0)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] +trio = ["trio (>=0.22.0,<1.0)"] + +[[package]] +name = "httpx" +version = "0.28.1" +description = "The next generation HTTP client." +optional = false +python-versions = ">=3.8" +files = [ + {file = "httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad"}, + {file = "httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc"}, +] + +[package.dependencies] +anyio = "*" +certifi = "*" +httpcore = "==1.*" +idna = "*" + +[package.extras] +brotli = ["brotli", "brotlicffi"] +cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "idna" +version = "3.10" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.6" +files = [ + {file = "idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"}, + {file = "idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9"}, +] + +[package.extras] +all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] + +[[package]] +name = "jiter" +version = "0.9.0" +description = "Fast iterable JSON parser." +optional = false +python-versions = ">=3.8" +files = [ + {file = "jiter-0.9.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:816ec9b60fdfd1fec87da1d7ed46c66c44ffec37ab2ef7de5b147b2fce3fd5ad"}, + {file = "jiter-0.9.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9b1d3086f8a3ee0194ecf2008cf81286a5c3e540d977fa038ff23576c023c0ea"}, + {file = "jiter-0.9.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1339f839b91ae30b37c409bf16ccd3dc453e8b8c3ed4bd1d6a567193651a4a51"}, + {file = "jiter-0.9.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ffba79584b3b670fefae66ceb3a28822365d25b7bf811e030609a3d5b876f538"}, + {file = "jiter-0.9.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5cfc7d0a8e899089d11f065e289cb5b2daf3d82fbe028f49b20d7b809193958d"}, + {file = "jiter-0.9.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e00a1a2bbfaaf237e13c3d1592356eab3e9015d7efd59359ac8b51eb56390a12"}, + {file = "jiter-0.9.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d1d9870561eb26b11448854dce0ff27a9a27cb616b632468cafc938de25e9e51"}, + {file = "jiter-0.9.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9872aeff3f21e437651df378cb75aeb7043e5297261222b6441a620218b58708"}, + {file = "jiter-0.9.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:1fd19112d1049bdd47f17bfbb44a2c0001061312dcf0e72765bfa8abd4aa30e5"}, + {file = "jiter-0.9.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6ef5da104664e526836070e4a23b5f68dec1cc673b60bf1edb1bfbe8a55d0678"}, + {file = "jiter-0.9.0-cp310-cp310-win32.whl", hash = "sha256:cb12e6d65ebbefe5518de819f3eda53b73187b7089040b2d17f5b39001ff31c4"}, + {file = "jiter-0.9.0-cp310-cp310-win_amd64.whl", hash = "sha256:c43ca669493626d8672be3b645dbb406ef25af3f4b6384cfd306da7eb2e70322"}, + {file = "jiter-0.9.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:6c4d99c71508912a7e556d631768dcdef43648a93660670986916b297f1c54af"}, + {file = "jiter-0.9.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8f60fb8ce7df529812bf6c625635a19d27f30806885139e367af93f6e734ef58"}, + {file = "jiter-0.9.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:51c4e1a4f8ea84d98b7b98912aa4290ac3d1eabfde8e3c34541fae30e9d1f08b"}, + {file = "jiter-0.9.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5f4c677c424dc76684fea3e7285a7a2a7493424bea89ac441045e6a1fb1d7b3b"}, + {file = "jiter-0.9.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2221176dfec87f3470b21e6abca056e6b04ce9bff72315cb0b243ca9e835a4b5"}, + {file = "jiter-0.9.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3c7adb66f899ffa25e3c92bfcb593391ee1947dbdd6a9a970e0d7e713237d572"}, + {file = "jiter-0.9.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c98d27330fdfb77913c1097a7aab07f38ff2259048949f499c9901700789ac15"}, + {file = "jiter-0.9.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:eda3f8cc74df66892b1d06b5d41a71670c22d95a1ca2cbab73654745ce9d0419"}, + {file = "jiter-0.9.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:dd5ab5ddc11418dce28343123644a100f487eaccf1de27a459ab36d6cca31043"}, + {file = "jiter-0.9.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:42f8a68a69f047b310319ef8e2f52fdb2e7976fb3313ef27df495cf77bcad965"}, + {file = "jiter-0.9.0-cp311-cp311-win32.whl", hash = "sha256:a25519efb78a42254d59326ee417d6f5161b06f5da827d94cf521fed961b1ff2"}, + {file = "jiter-0.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:923b54afdd697dfd00d368b7ccad008cccfeb1efb4e621f32860c75e9f25edbd"}, + {file = "jiter-0.9.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:7b46249cfd6c48da28f89eb0be3f52d6fdb40ab88e2c66804f546674e539ec11"}, + {file = "jiter-0.9.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:609cf3c78852f1189894383cf0b0b977665f54cb38788e3e6b941fa6d982c00e"}, + {file = "jiter-0.9.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d726a3890a54561e55a9c5faea1f7655eda7f105bd165067575ace6e65f80bb2"}, + {file = "jiter-0.9.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2e89dc075c1fef8fa9be219e249f14040270dbc507df4215c324a1839522ea75"}, + {file = "jiter-0.9.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:04e8ffa3c353b1bc4134f96f167a2082494351e42888dfcf06e944f2729cbe1d"}, + {file = "jiter-0.9.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:203f28a72a05ae0e129b3ed1f75f56bc419d5f91dfacd057519a8bd137b00c42"}, + {file = "jiter-0.9.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fca1a02ad60ec30bb230f65bc01f611c8608b02d269f998bc29cca8619a919dc"}, + {file = "jiter-0.9.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:237e5cee4d5d2659aaf91bbf8ec45052cc217d9446070699441a91b386ae27dc"}, + {file = "jiter-0.9.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:528b6b71745e7326eed73c53d4aa57e2a522242320b6f7d65b9c5af83cf49b6e"}, + {file = "jiter-0.9.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:9f48e86b57bc711eb5acdfd12b6cb580a59cc9a993f6e7dcb6d8b50522dcd50d"}, + {file = "jiter-0.9.0-cp312-cp312-win32.whl", hash = "sha256:699edfde481e191d81f9cf6d2211debbfe4bd92f06410e7637dffb8dd5dfde06"}, + {file = "jiter-0.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:099500d07b43f61d8bd780466d429c45a7b25411b334c60ca875fa775f68ccb0"}, + {file = "jiter-0.9.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:2764891d3f3e8b18dce2cff24949153ee30c9239da7c00f032511091ba688ff7"}, + {file = "jiter-0.9.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:387b22fbfd7a62418d5212b4638026d01723761c75c1c8232a8b8c37c2f1003b"}, + {file = "jiter-0.9.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40d8da8629ccae3606c61d9184970423655fb4e33d03330bcdfe52d234d32f69"}, + {file = "jiter-0.9.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a1be73d8982bdc278b7b9377426a4b44ceb5c7952073dd7488e4ae96b88e1103"}, + {file = "jiter-0.9.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2228eaaaa111ec54b9e89f7481bffb3972e9059301a878d085b2b449fbbde635"}, + {file = "jiter-0.9.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:11509bfecbc319459647d4ac3fd391d26fdf530dad00c13c4dadabf5b81f01a4"}, + {file = "jiter-0.9.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3f22238da568be8bbd8e0650e12feeb2cfea15eda4f9fc271d3b362a4fa0604d"}, + {file = "jiter-0.9.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:17f5d55eb856597607562257c8e36c42bc87f16bef52ef7129b7da11afc779f3"}, + {file = "jiter-0.9.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:6a99bed9fbb02f5bed416d137944419a69aa4c423e44189bc49718859ea83bc5"}, + {file = "jiter-0.9.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:e057adb0cd1bd39606100be0eafe742de2de88c79df632955b9ab53a086b3c8d"}, + {file = "jiter-0.9.0-cp313-cp313-win32.whl", hash = "sha256:f7e6850991f3940f62d387ccfa54d1a92bd4bb9f89690b53aea36b4364bcab53"}, + {file = "jiter-0.9.0-cp313-cp313-win_amd64.whl", hash = "sha256:c8ae3bf27cd1ac5e6e8b7a27487bf3ab5f82318211ec2e1346a5b058756361f7"}, + {file = "jiter-0.9.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f0b2827fb88dda2cbecbbc3e596ef08d69bda06c6f57930aec8e79505dc17001"}, + {file = "jiter-0.9.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:062b756ceb1d40b0b28f326cba26cfd575a4918415b036464a52f08632731e5a"}, + {file = "jiter-0.9.0-cp313-cp313t-win_amd64.whl", hash = "sha256:6f7838bc467ab7e8ef9f387bd6de195c43bad82a569c1699cb822f6609dd4cdf"}, + {file = "jiter-0.9.0-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:4a2d16360d0642cd68236f931b85fe50288834c383492e4279d9f1792e309571"}, + {file = "jiter-0.9.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e84ed1c9c9ec10bbb8c37f450077cbe3c0d4e8c2b19f0a49a60ac7ace73c7452"}, + {file = "jiter-0.9.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9f3c848209ccd1bfa344a1240763975ca917de753c7875c77ec3034f4151d06c"}, + {file = "jiter-0.9.0-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7825f46e50646bee937e0f849d14ef3a417910966136f59cd1eb848b8b5bb3e4"}, + {file = "jiter-0.9.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d82a811928b26d1a6311a886b2566f68ccf2b23cf3bfed042e18686f1f22c2d7"}, + {file = "jiter-0.9.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0c058ecb51763a67f019ae423b1cbe3fa90f7ee6280c31a1baa6ccc0c0e2d06e"}, + {file = "jiter-0.9.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9897115ad716c48f0120c1f0c4efae348ec47037319a6c63b2d7838bb53aaef4"}, + {file = "jiter-0.9.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:351f4c90a24c4fb8c87c6a73af2944c440494ed2bea2094feecacb75c50398ae"}, + {file = "jiter-0.9.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:d45807b0f236c485e1e525e2ce3a854807dfe28ccf0d013dd4a563395e28008a"}, + {file = "jiter-0.9.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:1537a890724ba00fdba21787010ac6f24dad47f763410e9e1093277913592784"}, + {file = "jiter-0.9.0-cp38-cp38-win32.whl", hash = "sha256:e3630ec20cbeaddd4b65513fa3857e1b7c4190d4481ef07fb63d0fad59033321"}, + {file = "jiter-0.9.0-cp38-cp38-win_amd64.whl", hash = "sha256:2685f44bf80e95f8910553bf2d33b9c87bf25fceae6e9f0c1355f75d2922b0ee"}, + {file = "jiter-0.9.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:9ef340fae98065071ccd5805fe81c99c8f80484e820e40043689cf97fb66b3e2"}, + {file = "jiter-0.9.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:efb767d92c63b2cd9ec9f24feeb48f49574a713870ec87e9ba0c2c6e9329c3e2"}, + {file = "jiter-0.9.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:113f30f87fb1f412510c6d7ed13e91422cfd329436364a690c34c8b8bd880c42"}, + {file = "jiter-0.9.0-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8793b6df019b988526f5a633fdc7456ea75e4a79bd8396a3373c371fc59f5c9b"}, + {file = "jiter-0.9.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7a9aaa5102dba4e079bb728076fadd5a2dca94c05c04ce68004cfd96f128ea34"}, + {file = "jiter-0.9.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d838650f6ebaf4ccadfb04522463e74a4c378d7e667e0eb1865cfe3990bfac49"}, + {file = "jiter-0.9.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0194f813efdf4b8865ad5f5c5f50f8566df7d770a82c51ef593d09e0b347020"}, + {file = "jiter-0.9.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a7954a401d0a8a0b8bc669199db78af435aae1e3569187c2939c477c53cb6a0a"}, + {file = "jiter-0.9.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4feafe787eb8a8d98168ab15637ca2577f6ddf77ac6c8c66242c2d028aa5420e"}, + {file = "jiter-0.9.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:27cd1f2e8bb377f31d3190b34e4328d280325ad7ef55c6ac9abde72f79e84d2e"}, + {file = "jiter-0.9.0-cp39-cp39-win32.whl", hash = "sha256:161d461dcbe658cf0bd0aa375b30a968b087cdddc624fc585f3867c63c6eca95"}, + {file = "jiter-0.9.0-cp39-cp39-win_amd64.whl", hash = "sha256:e8b36d8a16a61993be33e75126ad3d8aa29cf450b09576f3c427d27647fcb4aa"}, + {file = "jiter-0.9.0.tar.gz", hash = "sha256:aadba0964deb424daa24492abc3d229c60c4a31bfee205aedbf1acc7639d7893"}, +] + +[[package]] +name = "openai" +version = "1.76.0" +description = "The official Python library for the openai API" +optional = false +python-versions = ">=3.8" +files = [ + {file = "openai-1.76.0-py3-none-any.whl", hash = "sha256:a712b50e78cf78e6d7b2a8f69c4978243517c2c36999756673e07a14ce37dc0a"}, + {file = "openai-1.76.0.tar.gz", hash = "sha256:fd2bfaf4608f48102d6b74f9e11c5ecaa058b60dad9c36e409c12477dfd91fb2"}, +] + +[package.dependencies] +anyio = ">=3.5.0,<5" +distro = ">=1.7.0,<2" +httpx = ">=0.23.0,<1" +jiter = ">=0.4.0,<1" +pydantic = ">=1.9.0,<3" +sniffio = "*" +tqdm = ">4" +typing-extensions = ">=4.11,<5" + +[package.extras] +datalib = ["numpy (>=1)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)"] +realtime = ["websockets (>=13,<16)"] +voice-helpers = ["numpy (>=2.0.2)", "sounddevice (>=0.5.1)"] + +[[package]] +name = "pydantic" +version = "2.11.3" +description = "Data validation using Python type hints" +optional = false +python-versions = ">=3.9" +files = [ + {file = "pydantic-2.11.3-py3-none-any.whl", hash = "sha256:a082753436a07f9ba1289c6ffa01cd93db3548776088aa917cc43b63f68fa60f"}, + {file = "pydantic-2.11.3.tar.gz", hash = "sha256:7471657138c16adad9322fe3070c0116dd6c3ad8d649300e3cbdfe91f4db4ec3"}, +] + +[package.dependencies] +annotated-types = ">=0.6.0" +pydantic-core = "2.33.1" +typing-extensions = ">=4.12.2" +typing-inspection = ">=0.4.0" + +[package.extras] +email = ["email-validator (>=2.0.0)"] +timezone = ["tzdata"] + +[[package]] +name = "pydantic-core" +version = "2.33.1" +description = "Core functionality for Pydantic validation and serialization" +optional = false +python-versions = ">=3.9" +files = [ + {file = "pydantic_core-2.33.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:3077cfdb6125cc8dab61b155fdd714663e401f0e6883f9632118ec12cf42df26"}, + {file = "pydantic_core-2.33.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8ffab8b2908d152e74862d276cf5017c81a2f3719f14e8e3e8d6b83fda863927"}, + {file = "pydantic_core-2.33.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5183e4f6a2d468787243ebcd70cf4098c247e60d73fb7d68d5bc1e1beaa0c4db"}, + {file = "pydantic_core-2.33.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:398a38d323f37714023be1e0285765f0a27243a8b1506b7b7de87b647b517e48"}, + {file = "pydantic_core-2.33.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:87d3776f0001b43acebfa86f8c64019c043b55cc5a6a2e313d728b5c95b46969"}, + {file = "pydantic_core-2.33.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c566dd9c5f63d22226409553531f89de0cac55397f2ab8d97d6f06cfce6d947e"}, + {file = "pydantic_core-2.33.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0d5f3acc81452c56895e90643a625302bd6be351e7010664151cc55b7b97f89"}, + {file = "pydantic_core-2.33.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d3a07fadec2a13274a8d861d3d37c61e97a816beae717efccaa4b36dfcaadcde"}, + {file = "pydantic_core-2.33.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:f99aeda58dce827f76963ee87a0ebe75e648c72ff9ba1174a253f6744f518f65"}, + {file = "pydantic_core-2.33.1-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:902dbc832141aa0ec374f4310f1e4e7febeebc3256f00dc359a9ac3f264a45dc"}, + {file = "pydantic_core-2.33.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fe44d56aa0b00d66640aa84a3cbe80b7a3ccdc6f0b1ca71090696a6d4777c091"}, + {file = "pydantic_core-2.33.1-cp310-cp310-win32.whl", hash = "sha256:ed3eb16d51257c763539bde21e011092f127a2202692afaeaccb50db55a31383"}, + {file = "pydantic_core-2.33.1-cp310-cp310-win_amd64.whl", hash = "sha256:694ad99a7f6718c1a498dc170ca430687a39894a60327f548e02a9c7ee4b6504"}, + {file = "pydantic_core-2.33.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:6e966fc3caaf9f1d96b349b0341c70c8d6573bf1bac7261f7b0ba88f96c56c24"}, + {file = "pydantic_core-2.33.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bfd0adeee563d59c598ceabddf2c92eec77abcb3f4a391b19aa7366170bd9e30"}, + {file = "pydantic_core-2.33.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:91815221101ad3c6b507804178a7bb5cb7b2ead9ecd600041669c8d805ebd595"}, + {file = "pydantic_core-2.33.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9fea9c1869bb4742d174a57b4700c6dadea951df8b06de40c2fedb4f02931c2e"}, + {file = "pydantic_core-2.33.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d20eb4861329bb2484c021b9d9a977566ab16d84000a57e28061151c62b349a"}, + {file = "pydantic_core-2.33.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0fb935c5591573ae3201640579f30128ccc10739b45663f93c06796854405505"}, + {file = "pydantic_core-2.33.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c964fd24e6166420d18fb53996d8c9fd6eac9bf5ae3ec3d03015be4414ce497f"}, + {file = "pydantic_core-2.33.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:681d65e9011f7392db5aa002b7423cc442d6a673c635668c227c6c8d0e5a4f77"}, + {file = "pydantic_core-2.33.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e100c52f7355a48413e2999bfb4e139d2977a904495441b374f3d4fb4a170961"}, + {file = "pydantic_core-2.33.1-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:048831bd363490be79acdd3232f74a0e9951b11b2b4cc058aeb72b22fdc3abe1"}, + {file = "pydantic_core-2.33.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:bdc84017d28459c00db6f918a7272a5190bec3090058334e43a76afb279eac7c"}, + {file = "pydantic_core-2.33.1-cp311-cp311-win32.whl", hash = "sha256:32cd11c5914d1179df70406427097c7dcde19fddf1418c787540f4b730289896"}, + {file = "pydantic_core-2.33.1-cp311-cp311-win_amd64.whl", hash = "sha256:2ea62419ba8c397e7da28a9170a16219d310d2cf4970dbc65c32faf20d828c83"}, + {file = "pydantic_core-2.33.1-cp311-cp311-win_arm64.whl", hash = "sha256:fc903512177361e868bc1f5b80ac8c8a6e05fcdd574a5fb5ffeac5a9982b9e89"}, + {file = "pydantic_core-2.33.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:1293d7febb995e9d3ec3ea09caf1a26214eec45b0f29f6074abb004723fc1de8"}, + {file = "pydantic_core-2.33.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:99b56acd433386c8f20be5c4000786d1e7ca0523c8eefc995d14d79c7a081498"}, + {file = "pydantic_core-2.33.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:35a5ec3fa8c2fe6c53e1b2ccc2454398f95d5393ab398478f53e1afbbeb4d939"}, + {file = "pydantic_core-2.33.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b172f7b9d2f3abc0efd12e3386f7e48b576ef309544ac3a63e5e9cdd2e24585d"}, + {file = "pydantic_core-2.33.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9097b9f17f91eea659b9ec58148c0747ec354a42f7389b9d50701610d86f812e"}, + {file = "pydantic_core-2.33.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cc77ec5b7e2118b152b0d886c7514a4653bcb58c6b1d760134a9fab915f777b3"}, + {file = "pydantic_core-2.33.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d5e3d15245b08fa4a84cefc6c9222e6f37c98111c8679fbd94aa145f9a0ae23d"}, + {file = "pydantic_core-2.33.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ef99779001d7ac2e2461d8ab55d3373fe7315caefdbecd8ced75304ae5a6fc6b"}, + {file = "pydantic_core-2.33.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:fc6bf8869e193855e8d91d91f6bf59699a5cdfaa47a404e278e776dd7f168b39"}, + {file = "pydantic_core-2.33.1-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:b1caa0bc2741b043db7823843e1bde8aaa58a55a58fda06083b0569f8b45693a"}, + {file = "pydantic_core-2.33.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ec259f62538e8bf364903a7d0d0239447059f9434b284f5536e8402b7dd198db"}, + {file = "pydantic_core-2.33.1-cp312-cp312-win32.whl", hash = "sha256:e14f369c98a7c15772b9da98987f58e2b509a93235582838bd0d1d8c08b68fda"}, + {file = "pydantic_core-2.33.1-cp312-cp312-win_amd64.whl", hash = "sha256:1c607801d85e2e123357b3893f82c97a42856192997b95b4d8325deb1cd0c5f4"}, + {file = "pydantic_core-2.33.1-cp312-cp312-win_arm64.whl", hash = "sha256:8d13f0276806ee722e70a1c93da19748594f19ac4299c7e41237fc791d1861ea"}, + {file = "pydantic_core-2.33.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:70af6a21237b53d1fe7b9325b20e65cbf2f0a848cf77bed492b029139701e66a"}, + {file = "pydantic_core-2.33.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:282b3fe1bbbe5ae35224a0dbd05aed9ccabccd241e8e6b60370484234b456266"}, + {file = "pydantic_core-2.33.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b315e596282bbb5822d0c7ee9d255595bd7506d1cb20c2911a4da0b970187d3"}, + {file = "pydantic_core-2.33.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1dfae24cf9921875ca0ca6a8ecb4bb2f13c855794ed0d468d6abbec6e6dcd44a"}, + {file = "pydantic_core-2.33.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6dd8ecfde08d8bfadaea669e83c63939af76f4cf5538a72597016edfa3fad516"}, + {file = "pydantic_core-2.33.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2f593494876eae852dc98c43c6f260f45abdbfeec9e4324e31a481d948214764"}, + {file = "pydantic_core-2.33.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:948b73114f47fd7016088e5186d13faf5e1b2fe83f5e320e371f035557fd264d"}, + {file = "pydantic_core-2.33.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e11f3864eb516af21b01e25fac915a82e9ddad3bb0fb9e95a246067398b435a4"}, + {file = "pydantic_core-2.33.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:549150be302428b56fdad0c23c2741dcdb5572413776826c965619a25d9c6bde"}, + {file = "pydantic_core-2.33.1-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:495bc156026efafd9ef2d82372bd38afce78ddd82bf28ef5276c469e57c0c83e"}, + {file = "pydantic_core-2.33.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:ec79de2a8680b1a67a07490bddf9636d5c2fab609ba8c57597e855fa5fa4dacd"}, + {file = "pydantic_core-2.33.1-cp313-cp313-win32.whl", hash = "sha256:ee12a7be1742f81b8a65b36c6921022301d466b82d80315d215c4c691724986f"}, + {file = "pydantic_core-2.33.1-cp313-cp313-win_amd64.whl", hash = "sha256:ede9b407e39949d2afc46385ce6bd6e11588660c26f80576c11c958e6647bc40"}, + {file = "pydantic_core-2.33.1-cp313-cp313-win_arm64.whl", hash = "sha256:aa687a23d4b7871a00e03ca96a09cad0f28f443690d300500603bd0adba4b523"}, + {file = "pydantic_core-2.33.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:401d7b76e1000d0dd5538e6381d28febdcacb097c8d340dde7d7fc6e13e9f95d"}, + {file = "pydantic_core-2.33.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7aeb055a42d734c0255c9e489ac67e75397d59c6fbe60d155851e9782f276a9c"}, + {file = "pydantic_core-2.33.1-cp313-cp313t-win_amd64.whl", hash = "sha256:338ea9b73e6e109f15ab439e62cb3b78aa752c7fd9536794112e14bee02c8d18"}, + {file = "pydantic_core-2.33.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:5ab77f45d33d264de66e1884fca158bc920cb5e27fd0764a72f72f5756ae8bdb"}, + {file = "pydantic_core-2.33.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e7aaba1b4b03aaea7bb59e1b5856d734be011d3e6d98f5bcaa98cb30f375f2ad"}, + {file = "pydantic_core-2.33.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7fb66263e9ba8fea2aa85e1e5578980d127fb37d7f2e292773e7bc3a38fb0c7b"}, + {file = "pydantic_core-2.33.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3f2648b9262607a7fb41d782cc263b48032ff7a03a835581abbf7a3bec62bcf5"}, + {file = "pydantic_core-2.33.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:723c5630c4259400818b4ad096735a829074601805d07f8cafc366d95786d331"}, + {file = "pydantic_core-2.33.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d100e3ae783d2167782391e0c1c7a20a31f55f8015f3293647544df3f9c67824"}, + {file = "pydantic_core-2.33.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:177d50460bc976a0369920b6c744d927b0ecb8606fb56858ff542560251b19e5"}, + {file = "pydantic_core-2.33.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a3edde68d1a1f9af1273b2fe798997b33f90308fb6d44d8550c89fc6a3647cf6"}, + {file = "pydantic_core-2.33.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a62c3c3ef6a7e2c45f7853b10b5bc4ddefd6ee3cd31024754a1a5842da7d598d"}, + {file = "pydantic_core-2.33.1-cp39-cp39-musllinux_1_1_armv7l.whl", hash = "sha256:c91dbb0ab683fa0cd64a6e81907c8ff41d6497c346890e26b23de7ee55353f96"}, + {file = "pydantic_core-2.33.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:9f466e8bf0a62dc43e068c12166281c2eca72121dd2adc1040f3aa1e21ef8599"}, + {file = "pydantic_core-2.33.1-cp39-cp39-win32.whl", hash = "sha256:ab0277cedb698749caada82e5d099dc9fed3f906a30d4c382d1a21725777a1e5"}, + {file = "pydantic_core-2.33.1-cp39-cp39-win_amd64.whl", hash = "sha256:5773da0ee2d17136b1f1c6fbde543398d452a6ad2a7b54ea1033e2daa739b8d2"}, + {file = "pydantic_core-2.33.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5c834f54f8f4640fd7e4b193f80eb25a0602bba9e19b3cd2fc7ffe8199f5ae02"}, + {file = "pydantic_core-2.33.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:049e0de24cf23766f12cc5cc71d8abc07d4a9deb9061b334b62093dedc7cb068"}, + {file = "pydantic_core-2.33.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a28239037b3d6f16916a4c831a5a0eadf856bdd6d2e92c10a0da3a59eadcf3e"}, + {file = "pydantic_core-2.33.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d3da303ab5f378a268fa7d45f37d7d85c3ec19769f28d2cc0c61826a8de21fe"}, + {file = "pydantic_core-2.33.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:25626fb37b3c543818c14821afe0fd3830bc327a43953bc88db924b68c5723f1"}, + {file = "pydantic_core-2.33.1-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:3ab2d36e20fbfcce8f02d73c33a8a7362980cff717926bbae030b93ae46b56c7"}, + {file = "pydantic_core-2.33.1-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:2f9284e11c751b003fd4215ad92d325d92c9cb19ee6729ebd87e3250072cdcde"}, + {file = "pydantic_core-2.33.1-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:048c01eee07d37cbd066fc512b9d8b5ea88ceeb4e629ab94b3e56965ad655add"}, + {file = "pydantic_core-2.33.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:5ccd429694cf26af7997595d627dd2637e7932214486f55b8a357edaac9dae8c"}, + {file = "pydantic_core-2.33.1-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:3a371dc00282c4b84246509a5ddc808e61b9864aa1eae9ecc92bb1268b82db4a"}, + {file = "pydantic_core-2.33.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:f59295ecc75a1788af8ba92f2e8c6eeaa5a94c22fc4d151e8d9638814f85c8fc"}, + {file = "pydantic_core-2.33.1-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:08530b8ac922003033f399128505f513e30ca770527cc8bbacf75a84fcc2c74b"}, + {file = "pydantic_core-2.33.1-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bae370459da6a5466978c0eacf90690cb57ec9d533f8e63e564ef3822bfa04fe"}, + {file = "pydantic_core-2.33.1-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e3de2777e3b9f4d603112f78006f4ae0acb936e95f06da6cb1a45fbad6bdb4b5"}, + {file = "pydantic_core-2.33.1-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:3a64e81e8cba118e108d7126362ea30e021291b7805d47e4896e52c791be2761"}, + {file = "pydantic_core-2.33.1-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:52928d8c1b6bda03cc6d811e8923dffc87a2d3c8b3bfd2ce16471c7147a24850"}, + {file = "pydantic_core-2.33.1-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:1b30d92c9412beb5ac6b10a3eb7ef92ccb14e3f2a8d7732e2d739f58b3aa7544"}, + {file = "pydantic_core-2.33.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:f995719707e0e29f0f41a8aa3bcea6e761a36c9136104d3189eafb83f5cec5e5"}, + {file = "pydantic_core-2.33.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:7edbc454a29fc6aeae1e1eecba4f07b63b8d76e76a748532233c4c167b4cb9ea"}, + {file = "pydantic_core-2.33.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:ad05b683963f69a1d5d2c2bdab1274a31221ca737dbbceaa32bcb67359453cdd"}, + {file = "pydantic_core-2.33.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:df6a94bf9452c6da9b5d76ed229a5683d0306ccb91cca8e1eea883189780d568"}, + {file = "pydantic_core-2.33.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7965c13b3967909a09ecc91f21d09cfc4576bf78140b988904e94f130f188396"}, + {file = "pydantic_core-2.33.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3f1fdb790440a34f6ecf7679e1863b825cb5ffde858a9197f851168ed08371e5"}, + {file = "pydantic_core-2.33.1-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:5277aec8d879f8d05168fdd17ae811dd313b8ff894aeeaf7cd34ad28b4d77e33"}, + {file = "pydantic_core-2.33.1-pp39-pypy39_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:8ab581d3530611897d863d1a649fb0644b860286b4718db919bfd51ece41f10b"}, + {file = "pydantic_core-2.33.1-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:0483847fa9ad5e3412265c1bd72aad35235512d9ce9d27d81a56d935ef489672"}, + {file = "pydantic_core-2.33.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:de9e06abe3cc5ec6a2d5f75bc99b0bdca4f5c719a5b34026f8c57efbdecd2ee3"}, + {file = "pydantic_core-2.33.1.tar.gz", hash = "sha256:bcc9c6fdb0ced789245b02b7d6603e17d1563064ddcfc36f046b61c0c05dd9df"}, +] + +[package.dependencies] +typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" + +[[package]] +name = "sniffio" +version = "1.3.1" +description = "Sniff out which async library your code is running under" +optional = false +python-versions = ">=3.7" +files = [ + {file = "sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2"}, + {file = "sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"}, +] + +[[package]] +name = "tqdm" +version = "4.67.1" +description = "Fast, Extensible Progress Meter" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2"}, + {file = "tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[package.extras] +dev = ["nbval", "pytest (>=6)", "pytest-asyncio (>=0.24)", "pytest-cov", "pytest-timeout"] +discord = ["requests"] +notebook = ["ipywidgets (>=6)"] +slack = ["slack-sdk"] +telegram = ["requests"] + +[[package]] +name = "typing-extensions" +version = "4.13.2" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.13.2-py3-none-any.whl", hash = "sha256:a439e7c04b49fec3e5d3e2beaa21755cadbbdc391694e28ccdd36ca4a1408f8c"}, + {file = "typing_extensions-4.13.2.tar.gz", hash = "sha256:e6c81219bd689f51865d9e372991c540bda33a0379d5573cddb9a3a23f7caaef"}, +] + +[[package]] +name = "typing-inspection" +version = "0.4.0" +description = "Runtime typing introspection tools" +optional = false +python-versions = ">=3.9" +files = [ + {file = "typing_inspection-0.4.0-py3-none-any.whl", hash = "sha256:50e72559fcd2a6367a19f7a7e610e6afcb9fac940c650290eed893d61386832f"}, + {file = "typing_inspection-0.4.0.tar.gz", hash = "sha256:9765c87de36671694a67904bf2c96e395be9c6439bb6c87b5142569dcdd65122"}, +] + +[package.dependencies] +typing-extensions = ">=4.12.0" + +[metadata] +lock-version = "2.0" +python-versions = "^3.12" +content-hash = "1c3756b345de08cbaabc8bdace2ff3f900bd1f50dffbd0f21ea1a0d0f23ea7cf" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..4acc469 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,16 @@ +[tool.poetry] +name = "pynewsscraper" +version = "0.1.0" +description = "" +authors = ["Daniel Soares Martins "] +readme = "README.md" + +[tool.poetry.dependencies] +python = "^3.12" +babel = "^2.17.0" +openai = "^1.76.0" + + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/requirements.txt b/requirements.txt index e8897ea..6370920 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ babel==2.15.0 crawl4ai==0.4.247 -google-generativeai==0.8.4 \ No newline at end of file +google-generativeai==0.8.4 +playwright==1.51.0