fetchtest.py

# fetch.py
import urllib.request
import xml.etree.ElementTree as ET
import json

def fetch(category, n=200):
    url = f"http://export.arxiv.org/api/query?search_query=cat:{category}&start=0&max_results={n}"
    root = ET.fromstring(urllib.request.urlopen(url).read())
    ns = '{http://www.w3.org/2005/Atom}'
    papers = []
    for entry in root.findall(f'{ns}entry'):
        title = entry.find(f'{ns}title').text.strip()
        abstract = entry.find(f'{ns}summary').text.strip()
        papers.append({"text": f"{title}\n\n{abstract}"})
    return papers

papers = []
for cat in ["eess.SP", "cs.IT", "cs.NI"]:
    papers.extend(fetch(cat))

with open("wireless.jsonl", "w") as f:
    for p in papers:
        f.write(json.dumps(p) + "\n")

print(f"Done! {len(papers)} papers")